bookorbit/koreader-plugin/bookorbit.koplugin/bookorbit_sidecar.lua

--[[--
Extracts syncable data from a book's sidecar without opening the document.

The normalizers are shared with the per-book snapshot sync, which feeds them
the live in-memory annotation list and summary table instead of sidecar data
(both have the same shape), so server-side annotation keys stay stable no
matter which path uploaded them.
]]

local BookOrbitCapabilities = require("bookorbit_capabilities")
local DocSettings = require("docsettings")
local BookList = require("ui/widget/booklist")
local lfs = require("libs/libkoreader-lfs")

local ALLOWED_DRAWERS = {
    lighten = true,
    underscore = true,
    strikeout = true,
    invert = true,
}

local ALLOWED_STATUSES = {
    reading = true,
    complete = true,
    abandoned = true,
}

local BookOrbitSidecar = {}

-- Written into a sidecar once it has been through a complete exchange. A sidecar
-- without it is new to the server (a re-download, a deleted .sdr), and its empty
-- key list must not read as "the reader deleted everything".
BookOrbitSidecar.EXCHANGE_MARKER = "bookorbit_exchanged_at"

function BookOrbitSidecar.isMarked(doc_settings)
    if not doc_settings then return nil end
    return doc_settings:readSetting(BookOrbitSidecar.EXCHANGE_MARKER) ~= nil
end

function BookOrbitSidecar.readMarked(file)
    if not file or not DocSettings:hasSidecarFile(file) then return false end
    return BookOrbitSidecar.isMarked(DocSettings:open(file))
end

-- The open book's settings are flushed by KOReader itself.
function BookOrbitSidecar.markLive(doc_settings, now)
    if not doc_settings or BookOrbitSidecar.isMarked(doc_settings) then return false end
    doc_settings:saveSetting(BookOrbitSidecar.EXCHANGE_MARKER, now or os.time())
    return true
end

function BookOrbitSidecar.markFile(file, now)
    if not file or lfs.attributes(file, "mode") ~= "file" then return false end
    local doc_settings = DocSettings:open(file)
    if BookOrbitSidecar.isMarked(doc_settings) then return false end
    doc_settings:saveSetting(BookOrbitSidecar.EXCHANGE_MARKER, now or os.time())
    doc_settings:flush()
    return true
end

-- What the exchange tells the server about this sidecar: "known" once marked,
-- "new" when unmarked and holding no highlight or dogear at all, nil (let the
-- server judge from its key set) when unmarked but holding entries, as every
-- sidecar written before the marker existed does.
function BookOrbitSidecar.sidecarState(marked, local_count)
    if marked == nil then return nil end
    if marked then return "known" end
    if (local_count or 0) == 0 then return "new" end
    return nil
end

BookOrbitSidecar.SIDECAR_CAPABILITY = "sidecarState"

-- Only a server that advertises sidecarState accepts the field. An older one
-- would read a new sidecar's empty key list as "everything was deleted", so
-- there the list goes out as incomplete instead and nothing is deleted.
function BookOrbitSidecar.resolveExchangeState(client, sidecar_state, keys_complete)
    if sidecar_state == nil then return nil, keys_complete end
    if client and client.getPluginVersion
            and BookOrbitCapabilities.supports(client, BookOrbitSidecar.SIDECAR_CAPABILITY) == true then
        return sidecar_state, keys_complete
    end
    if sidecar_state == "new" then return nil, false end
    return nil, keys_complete
end

function BookOrbitSidecar.sidecarMtime(file)
    local sidecar_file = DocSettings:findSidecarFile(file)
    if not sidecar_file then return nil end
    return lfs.attributes(sidecar_file, "modification")
end

local function isDeviceDatetime(value)
    return type(value) == "string" and value:match("^%d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d$") ~= nil
end

local function isDateOnly(value)
    return type(value) == "string" and value:match("^%d%d%d%d%-%d%d%-%d%d$") ~= nil
end

local function truncate(value, max_len)
    if type(value) ~= "string" then return nil end
    if #value > max_len then
        return value:sub(1, max_len)
    end
    return value
end

-- PDF positions are tables; serialize them canonically by hand so the
-- server-side annotation key (md5 of datetime|pos0) stays stable across syncs.
local function serializePos(pos)
    if type(pos) == "string" then
        return pos, "xpointer"
    end
    if type(pos) == "table" and type(pos.page) == "number" then
        return string.format('{"page":%d,"x":%.2f,"y":%.2f}', pos.page, tonumber(pos.x) or 0, tonumber(pos.y) or 0), "pdf"
    end
    return nil
end
BookOrbitSidecar.serializePos = serializePos

-- Order-independent change signal for one normalized entry. Count plus max
-- datetime cannot see a delete-plus-add that leaves both unchanged, so every
-- entry contributes a hash of its identity fields. Positions can be kilobytes
-- long, so only their length and both ends are sampled: this signal decides
-- whether an unchanged book may skip an exchange, never what gets uploaded.
-- The note is hashed whole because rewriting it leaves every timestamp alone.
local function entryHash(entry)
    local pos0 = entry.pos0 or ""
    local key = entry.datetime .. "|" .. #pos0 .. "|" .. pos0:sub(1, 24) .. "|" .. pos0:sub(-24)
        .. "|" .. (entry.datetimeUpdated or "") .. "|" .. (entry.note or "")
    local hash = 5381
    for index = 1, #key do
        hash = (hash * 33 + key:byte(index)) % 4294967296
    end
    return hash
end

-- Returns normalized annotations, the max effective datetime and a signature
-- that changes whenever the local set changes; accepts the raw sidecar list or
-- the live ui.annotation.annotations array.
function BookOrbitSidecar.normalizeAnnotations(raw)
    local annotations = {}
    local max_datetime = ""
    local sum, mix = 0, 0
    for _, a in ipairs(raw or {}) do
        -- drawer == nil marks a position-only bookmark; those are skipped in v1.
        if not a.bookorbit_pending_verification and ALLOWED_DRAWERS[a.drawer] and isDeviceDatetime(a.datetime) then
            local pos0, pos_format = serializePos(a.pos0)
            if pos0 then
                local pos1 = serializePos(a.pos1)
                local entry = {
                    datetime = a.datetime,
                    datetimeUpdated = isDeviceDatetime(a.datetime_updated) and a.datetime_updated or nil,
                    drawer = a.drawer,
                    color = truncate(a.color, 30),
                    text = truncate(a.text, 10000),
                    note = truncate(a.note, 5000),
                    chapter = truncate(a.chapter, 500),
                    pageno = type(a.pageno) == "number" and math.floor(a.pageno) or nil,
                    posFormat = pos_format,
                    pos0 = truncate(pos0, 4000),
                    pos1 = truncate(pos1, 4000),
                }
                table.insert(annotations, entry)
                local effective = entry.datetimeUpdated or entry.datetime
                if effective > max_datetime then
                    max_datetime = effective
                end
                local hash = entryHash(entry)
                sum = (sum + hash) % 4294967296
                mix = (mix + hash * (hash % 8191 + 1)) % 4294967296
            end
        end
    end
    return annotations, max_datetime,
        string.format("%d:%s:%d:%d", #annotations, max_datetime, sum, mix)
end

-- Position-only bookmarks carry no text, so a rename is only visible in the
-- fields themselves. KOReader also leaves datetime_updated untouched when the
-- user edits a dogear note, which is why the note and chapter are hashed here
-- and why the exchange uploads the whole set rather than a datetime delta.
local function bookmarkHash(entry)
    local pos = entry.pos or ""
    local key = entry.datetime .. "|" .. #pos .. "|" .. pos:sub(1, 24) .. "|" .. pos:sub(-24)
        .. "|" .. (entry.datetimeUpdated or "") .. "|" .. (entry.note or "") .. "|" .. (entry.chapter or "")
    local hash = 5381
    for index = 1, #key do
        hash = (hash * 33 + key:byte(index)) % 4294967296
    end
    return hash
end

-- Returns normalized position-only bookmarks (dogears), the max effective
-- datetime and a signature that changes whenever the local set changes.
-- Paging documents store a page number instead of an xpointer; those are out of
-- scope, so only string positions are kept.
function BookOrbitSidecar.normalizeBookmarks(raw)
    local bookmarks = {}
    local max_datetime = ""
    local sum, mix = 0, 0
    for _, a in ipairs(raw or {}) do
        if a.drawer == nil and isDeviceDatetime(a.datetime) and type(a.page) == "string" and a.page ~= "" then
            local entry = {
                datetime = a.datetime,
                datetimeUpdated = isDeviceDatetime(a.datetime_updated) and a.datetime_updated or nil,
                pos = truncate(a.page, 4000),
                pageno = type(a.pageno) == "number" and math.floor(a.pageno) or nil,
                chapter = truncate(a.chapter, 500),
                note = truncate(a.note, 500),
            }
            table.insert(bookmarks, entry)
            local effective = entry.datetimeUpdated or entry.datetime
            if effective > max_datetime then
                max_datetime = effective
            end
            local hash = bookmarkHash(entry)
            sum = (sum + hash) % 4294967296
            mix = (mix + hash * (hash % 8191 + 1)) % 4294967296
        end
    end
    return bookmarks, max_datetime,
        string.format("%d:%s:%d:%d", #bookmarks, max_datetime, sum, mix)
end

-- Validates a summary table (sidecar or live doc_settings reference) into a
-- fresh plain table; never returns the input reference.
function BookOrbitSidecar.normalizeSummary(summary)
    if type(summary) ~= "table" then summary = {} end
    local rating = nil
    if type(summary.rating) == "number" and summary.rating >= 1 and summary.rating <= 5 then
        rating = math.floor(summary.rating)
    end
    local review_note = truncate(summary.note, 10000)
    return {
        status = ALLOWED_STATUSES[summary.status] and summary.status or nil,
        status_modified = isDateOnly(summary.modified) and summary.modified or nil,
        rating = rating,
        review_note = review_note,
    }
end

local function dateKey(value)
    if type(value) ~= "string" then return nil end
    local key = value:sub(1, 10)
    return isDateOnly(key) and key or nil
end

local function maxDateKey(a, b)
    if not a then return b end
    if not b then return a end
    return a >= b and a or b
end

function BookOrbitSidecar.stateFromServerResult(result)
    if type(result) ~= "table" then return nil end
    local has_status = result.statusSet == true and ALLOWED_STATUSES[result.status] == true
    local has_rating = type(result.ratingSet) == "boolean"
    local has_review = type(result.reviewNoteSet) == "boolean"
    if not has_status and not has_rating and not has_review then return nil end

    local state = {
        status_known = has_status,
        status = has_status and result.status or nil,
        status_modified = has_status and dateKey(result.statusUpdatedAt) or nil,
        rating_known = has_rating,
        rating = has_rating and result.ratingSet and type(result.rating) == "number" and math.floor(result.rating) or nil,
        review_known = has_review,
        review_note = has_review and result.reviewNoteSet and truncate(result.reviewNote, 10000) or nil,
        modified = maxDateKey(dateKey(result.ratingUpdatedAt), dateKey(result.reviewUpdatedAt)),
    }
    return state
end

function BookOrbitSidecar.rememberServerState(book, state)
    if not book or not state then return end
    if state.status_known then
        book.statusSyncedKnown = true
        book.statusSynced = state.status
        book.statusSyncedModified = state.status_modified
    end
    if state.rating_known then
        book.ratingSyncedKnown = true
        book.ratingSynced = state.rating
    end
    if state.review_known then
        book.reviewSyncedKnown = true
        book.reviewSyncedNote = state.review_note
    end
end

function BookOrbitSidecar.rememberUploadedState(book, summary, payload)
    if not book or not payload then return end
    if payload.status then
        book.statusSyncedKnown = true
        book.statusSynced = payload.status
        book.statusSyncedModified = payload.statusModified
    end
    if payload.rating ~= nil or payload.ratingCleared then
        book.ratingSyncedKnown = true
        book.ratingSynced = summary.rating
    end
    if payload.reviewNote ~= nil or payload.reviewCleared then
        book.reviewSyncedKnown = true
        book.reviewSyncedNote = summary.review_note
    end
end

function BookOrbitSidecar.buildStatePayload(hash, book, summary, force_pull)
    local status_changed = summary.status ~= nil
        and (book.statusSyncedKnown ~= true
            or summary.status ~= book.statusSynced)

    local rating_known = book.ratingSyncedKnown == true or book.ratingSynced ~= nil
    local rating_changed = false
    if summary.rating ~= nil then
        rating_changed = not rating_known or summary.rating ~= book.ratingSynced
    elseif rating_known and book.ratingSynced ~= nil then
        rating_changed = true
    end

    local review_known = book.reviewSyncedKnown == true or book.reviewSyncedNote ~= nil
    local review_changed = false
    if summary.review_note ~= nil then
        review_changed = not review_known or summary.review_note ~= book.reviewSyncedNote
    elseif review_known and book.reviewSyncedNote ~= nil then
        review_changed = true
    end

    if not status_changed and not rating_changed and not review_changed then
        return force_pull and { hash = hash } or nil
    end

    local payload = { hash = hash }
    if status_changed then
        payload.status = summary.status
        payload.statusModified = summary.status_modified
    end
    if rating_changed then
        if summary.rating ~= nil then
            payload.rating = summary.rating
        else
            payload.ratingCleared = true
        end
        payload.statusModified = payload.statusModified or summary.status_modified
    end
    if review_changed then
        if summary.review_note ~= nil then
            payload.reviewNote = summary.review_note
        else
            payload.reviewCleared = true
        end
        payload.reviewModified = summary.status_modified
    end
    return payload, status_changed, rating_changed, review_changed
end

-- A forced pull is the only way the device learns a rating or review written
-- on the web, but paying for it on every drained lifecycle entry is what made
-- an unchanged book still cost a request. Bound it by age instead: manual sync
-- and the sweep pull unconditionally, everything else once per interval.
BookOrbitSidecar.STATE_PULL_MAX_AGE = 24 * 3600

function BookOrbitSidecar.needsStatePull(book, now)
    local pulled_at = book and book.statePulledAt
    if type(pulled_at) ~= "number" then return true end
    now = now or os.time()
    if pulled_at > now then return true end
    return (now - pulled_at) > BookOrbitSidecar.STATE_PULL_MAX_AGE
end

function BookOrbitSidecar.markStatePulled(book, now)
    if not book then return end
    book.statePulledAt = now or os.time()
end

local function applySummary(doc_settings, state)
    if not doc_settings or not state then return false end
    local summary = doc_settings:readSetting("summary") or {}
    local touched = false
    if state.status_known and summary.status ~= state.status then
        summary.status = state.status
        touched = true
    end
    if state.rating_known and summary.rating ~= state.rating then
        summary.rating = state.rating
        touched = true
    end
    if state.review_known and summary.note ~= state.review_note then
        summary.note = state.review_note
        touched = true
    end
    if touched then
        summary.modified = maxDateKey(state.status_modified, state.modified) or os.date("%Y-%m-%d", os.time())
        doc_settings:saveSetting("summary", summary)
    end
    return touched
end

function BookOrbitSidecar.applyServerStateLive(ui, state)
    if not ui or not ui.doc_settings then return false end
    local touched = applySummary(ui.doc_settings, state)
    if touched and ui.document and ui.document.file then
        if state.status_known then BookList.setBookInfoCacheProperty(ui.document.file, "status", state.status) end
        if state.rating_known then BookList.setBookInfoCacheProperty(ui.document.file, "rating", state.rating) end
    end
    return touched
end

function BookOrbitSidecar.applyServerStateSidecar(file, state)
    if not file or not state then return false end
    local doc_settings = DocSettings:open(file)
    local touched = applySummary(doc_settings, state)
    if touched then
        doc_settings:flush()
        if state.status_known then BookList.setBookInfoCacheProperty(file, "status", state.status) end
        if state.rating_known then BookList.setBookInfoCacheProperty(file, "rating", state.rating) end
    end
    return touched
end

function BookOrbitSidecar.extract(file)
    if not DocSettings:hasSidecarFile(file) then return nil end

    local doc_settings = DocSettings:open(file)
    local summary = BookOrbitSidecar.normalizeSummary(doc_settings:readSetting("summary"))
    local raw_annotations = doc_settings:readSetting("annotations")
    local annotations, max_datetime, signature = BookOrbitSidecar.normalizeAnnotations(raw_annotations)
    local bookmarks, bm_max_datetime, bm_signature = BookOrbitSidecar.normalizeBookmarks(raw_annotations)

    local last_position = doc_settings:readSetting("last_xpointer")
    if not last_position then
        local last_page = doc_settings:readSetting("last_page")
        if last_page then
            last_position = tostring(last_page)
        end
    end

    return {
        md5 = doc_settings:readSetting("partial_md5_checksum"),
        percent_finished = doc_settings:readSetting("percent_finished"),
        last_position = last_position,
        status = summary.status,
        status_modified = summary.status_modified,
        rating = summary.rating,
        review_note = summary.review_note,
        annotations = annotations,
        annotations_count = #annotations,
        annotations_max_datetime = max_datetime,
        annotations_signature = signature,
        bookmarks = bookmarks,
        bookmarks_count = #bookmarks,
        bookmarks_max_datetime = bm_max_datetime,
        bookmarks_signature = bm_signature,
        marked = BookOrbitSidecar.isMarked(doc_settings),
    }
end

return BookOrbitSidecar