bookorbit/koreader-plugin/bookorbit.koplugin/bookorbit_bulk_checkpoint.lua

--[[--
Resumable checkpoint for a bulk download run.

One compact record, published with an atomic rename so an interruption can
never leave a half-written checkpoint. It holds the source identity, the
manifest snapshot it was enumerated against, the last fully committed cursor
and bounded per-page completion and failure summaries. It deliberately does not
retain manifest rows: a resumed run re-enumerates from the cursor and validates
already-published files instead.
]]

local DataStorage = require("datastorage")
local dump = require("dump")
local lfs = require("libs/libkoreader-lfs")
local util = require("util")

local CHECKPOINT_DIR = "bookorbit-bulk-download"
local CHECKPOINT_FILE = "checkpoint.lua"
local VERSION = 2
local MAX_COMPLETED = 250
local MAX_FAILURES = 50

local Checkpoint = {}
Checkpoint.__index = Checkpoint

Checkpoint.VERSION = VERSION
Checkpoint.MAX_COMPLETED = MAX_COMPLETED
Checkpoint.MAX_FAILURES = MAX_FAILURES

local function safeLoad(path)
    local chunk = loadfile(path)
    if not chunk then return nil end
    setfenv(chunk, {})
    local ok, value = pcall(chunk)
    if not ok or type(value) ~= "table" then return nil end
    return value
end

function Checkpoint.open(opts)
    opts = opts or {}
    local settings_dir = opts.settings_dir or DataStorage:getSettingsDir()
    local dir = settings_dir .. "/" .. CHECKPOINT_DIR
    return setmetatable({
        dir = dir,
        path = dir .. "/" .. CHECKPOINT_FILE,
        now = opts.now or os.time,
        rename = opts.rename or os.rename,
        remove = opts.remove or os.remove,
        write_file = opts.write_file or function(content, path)
            return util.writeToFile(content, path, true, false, true)
        end,
        make_path = opts.make_path or util.makePath,
        attributes = opts.attributes or lfs.attributes,
        load_record = opts.load_record or safeLoad,
        serialize = opts.serialize or function(record)
            return "return " .. dump(record) .. "\n"
        end,
    }, Checkpoint)
end

function Checkpoint:ensureDirectory()
    if self.attributes(self.dir, "mode") == "directory" then return true end
    self.make_path(self.dir)
    return self.attributes(self.dir, "mode") == "directory"
end

function Checkpoint:load()
    local record = self.load_record(self.path)
    if type(record) ~= "table" or record.version ~= VERSION then return nil end
    if type(record.source_key) ~= "string" or record.source_key == "" then return nil end
    record.completed = type(record.completed) == "table" and record.completed or {}
    record.linked_paths = type(record.linked_paths) == "table" and record.linked_paths or {}
    record.counts = type(record.counts) == "table" and record.counts or {}
    record.failures = type(record.failures) == "table" and record.failures or {}
    return record
end

local function boundedCompleted(completed)
    local bounded, kept = {}, 0
    for book_id, path in pairs(completed or {}) do
        if kept >= MAX_COMPLETED then break end
        bounded[book_id] = path
        kept = kept + 1
    end
    return bounded
end

local function boundedFailures(failures)
    local bounded = {}
    for index, failure in ipairs(failures or {}) do
        if index > MAX_FAILURES then break end
        table.insert(bounded, { id = failure.id, title = failure.title })
    end
    return bounded
end

function Checkpoint:save(record)
    if not self:ensureDirectory() then return nil, "directory_failed" end
    local stored = {
        version = VERSION,
        source_key = record.source_key,
        label = record.label,
        manifest_version = record.manifest_version,
        cursor = record.cursor,
        chunk_index = record.chunk_index,
        page_number = record.page_number,
        processed = record.processed,
        completed = boundedCompleted(record.completed),
        linked_paths = boundedCompleted(record.linked_paths),
        counts = record.counts,
        failures = boundedFailures(record.failures),
        updated_at = self.now(),
    }

    local temporary = self.path .. ".tmp"
    self.remove(temporary)
    local written, write_err = self.write_file(self.serialize(stored), temporary)
    if not written then
        self.remove(temporary)
        return nil, write_err or "write_failed"
    end
    local renamed, rename_err = self.rename(temporary, self.path)
    if not renamed then
        self.remove(temporary)
        return nil, rename_err or "rename_failed"
    end
    return stored
end

function Checkpoint:clear()
    self.remove(self.path .. ".tmp")
    self.remove(self.path)
end

return Checkpoint