fix(stats): route KOReader hash collisions safely

Track colliding KOReader statistics rows independently and require verified explicit targets before upload. Keep unresolved history pending with a visible recovery path.

Fixes #1457
This commit is contained in:
neon
2026-09-23 03:27:52 +00:00
parent 27cfdc2028
commit 94eca198c3
19 changed files with 566 additions and 42 deletions
@@ -95,7 +95,14 @@ function BookOrbitBookSync.capture(plugin)
end end
metadata = metadata or {} metadata = metadata or {}
local current_stats_id = stats and tonumber(stats.id_curr_book) or nil local current_stats_id = stats and tonumber(stats.id_curr_book) or nil
local stats_ambiguous = current_stats_id == nil and metadata.metadata_ambiguous == true local stats_ambiguous = current_stats_id == nil and metadata.stats_ambiguous == true
local current_stats_row
for _, row in ipairs(metadata.rows or {}) do
if row.id == current_stats_id then
current_stats_row = row
break
end
end
local stats_ids = {} local stats_ids = {}
if current_stats_id then if current_stats_id then
table.insert(stats_ids, current_stats_id) table.insert(stats_ids, current_stats_id)
@@ -114,6 +121,8 @@ function BookOrbitBookSync.capture(plugin)
last_open = metadata.last_open or ts, last_open = metadata.last_open or ts,
metadata_ambiguous = false, metadata_ambiguous = false,
stats_metadata_ambiguous = stats_ambiguous, stats_metadata_ambiguous = stats_ambiguous,
stats_row_ambiguous = current_stats_row ~= nil and metadata.stats_ambiguous == true,
stats_row = current_stats_row,
stats_identity_repaired = plugin.bookorbit_document_digest stats_identity_repaired = plugin.bookorbit_document_digest
and plugin.bookorbit_document_digest.repaired == true or false, and plugin.bookorbit_document_digest.repaired == true or false,
stats_ids = stats_ids, stats_ids = stats_ids,
@@ -279,6 +288,21 @@ end
local stepMatch, stepStats, stepAnnotations, stepAnnotationsLegacy, stepBookmarks, stepState, stepProgress local stepMatch, stepStats, stepAnnotations, stepAnnotationsLegacy, stepBookmarks, stepState, stepProgress
local function bindStatsRow(ctx, book)
local row = ctx.snap.stats_row
if not row or not ctx.snap.stats_row_ambiguous then return nil end
return ctx.state:bindStatsRow(
row.id, ctx.snap.digest, row.title, row.authors, book.fileId, book.bookId)
end
local function matchedFileCandidate(ctx, book)
if not ctx.snap.stats_row_ambiguous then return book and book.fileId or nil end
local row = ctx.snap.stats_row
if not row then return nil end
local target = ctx.state:getStatsRow(row.id, ctx.snap.digest, row.title, row.authors)
return target and target.bookFileId or nil
end
stepMatch = function(ctx) stepMatch = function(ctx)
if ctx.acknowledged.match then if ctx.acknowledged.match then
return step(ctx, stepStats) return step(ctx, stepStats)
@@ -288,7 +312,10 @@ stepMatch = function(ctx)
book.file = ctx.snap.file book.file = ctx.snap.file
end end
if not ctx.force_match and BookOrbitState.isMatchFresh(book, ctx.state.global) then if not ctx.force_match
and not ctx.snap.stats_row_ambiguous
and BookOrbitState.isMatchFresh(book, ctx.state.global) then
bindStatsRow(ctx, book)
logger.dbg("BookOrbit: book sync match skipped, verified match still fresh") logger.dbg("BookOrbit: book sync match skipped, verified match still fresh")
if not acknowledge(ctx, "match", false) then return end if not acknowledge(ctx, "match", false) then return end
return step(ctx, stepStats) return step(ctx, stepStats)
@@ -300,7 +327,7 @@ stepMatch = function(ctx)
authors = ctx.snap.authors, authors = ctx.snap.authors,
last_open = ctx.snap.last_open, last_open = ctx.snap.last_open,
source = "current_file", source = "current_file",
book_file_id = book and book.fileId or nil, book_file_id = matchedFileCandidate(ctx, book),
metadata_ambiguous = ctx.snap.metadata_ambiguous, metadata_ambiguous = ctx.snap.metadata_ambiguous,
}, },
}) })
@@ -317,6 +344,7 @@ stepMatch = function(ctx)
for _, match in ipairs(body.matches or {}) do for _, match in ipairs(body.matches or {}) do
if match.hash == ctx.snap.digest then if match.hash == ctx.snap.digest then
ctx.state:setMatched(match.hash, match.bookFileId, match.bookId, ctx.snap.file) ctx.state:setMatched(match.hash, match.bookFileId, match.bookId, ctx.snap.file)
bindStatsRow(ctx, ctx.state:getBook(ctx.snap.digest))
if not acknowledge(ctx, "match") then return end if not acknowledge(ctx, "match") then return end
return step(ctx, stepStats) return step(ctx, stepStats)
end end
@@ -352,8 +380,9 @@ stepStats = function(ctx)
return step(ctx, stepAnnotations) return step(ctx, stepAnnotations)
end end
local stats_target = bindStatsRow(ctx, book) or book
if ctx.stats_watermark == nil then if ctx.stats_watermark == nil then
ctx.stats_watermark = ctx.snap.stats_identity_repaired and 0 or (book.statsWatermark or 0) ctx.stats_watermark = ctx.snap.stats_identity_repaired and 0 or (stats_target.statsWatermark or 0)
end end
local watermark = ctx.stats_watermark local watermark = ctx.stats_watermark
local events = BookOrbitStatsReader.getEventsAfter(ctx.stat_ids, watermark, STATS_BATCH) local events = BookOrbitStatsReader.getEventsAfter(ctx.stat_ids, watermark, STATS_BATCH)
@@ -362,7 +391,9 @@ stepStats = function(ctx)
return step(ctx, stepAnnotations) return step(ctx, stepAnnotations)
end end
local body, err = ctx.client:uploadPageStats({ { hash = ctx.snap.digest, events = events } }) local upload = { hash = ctx.snap.digest, events = events }
if ctx.snap.stats_row_ambiguous then upload.bookFileId = stats_target.bookFileId end
local body, err = ctx.client:uploadPageStats({ upload })
if not body then if not body then
if isAuthError(err) then return finish(ctx, "auth") end if isAuthError(err) then return finish(ctx, "auth") end
ctx.had_errors = true ctx.had_errors = true
@@ -375,8 +406,8 @@ stepStats = function(ctx)
return finish(ctx, "unmatched") return finish(ctx, "unmatched")
end end
local more = BookOrbitState.applyStatsAck(book, events, body, ctx.snap.digest, STATS_BATCH, watermark) local more = BookOrbitState.applyStatsAck(stats_target, events, body, ctx.snap.digest, STATS_BATCH, watermark)
ctx.stats_watermark = book.statsWatermark or watermark ctx.stats_watermark = stats_target.statsWatermark or watermark
ctx.counts.page_stats = ctx.counts.page_stats + #events ctx.counts.page_stats = ctx.counts.page_stats + #events
if more then if more then
return step(ctx, stepStats) return step(ctx, stepStats)
@@ -780,6 +811,7 @@ function BookOrbitBookSync.run(opts)
state = BookOrbitStateManager.session({ state = BookOrbitStateManager.session({
digests = { snap.digest }, digests = { snap.digest },
files = { snap.file }, files = { snap.file },
statsRows = snap.stats_ids,
}), }),
snap = snap, snap = snap,
reason = opts.reason or "manual", reason = opts.reason or "manual",
@@ -406,10 +406,14 @@ function Outbox:enqueue(snapshot, opts)
end end
local status = self:status() local status = self:status()
local existing local existing, existing_with_payload
for _, entry in ipairs(self:listMetadata()) do for _, entry in ipairs(self:listMetadata()) do
if entry.digest == snapshot.digest and entry.id ~= self.active_id then if entry.digest == snapshot.digest and entry.id ~= self.active_id then
existing = entry local candidate = self:attachPayload(entry)
if candidate and candidate.snapshot.file == snapshot.file then
existing = entry
existing_with_payload = candidate
end
end end
end end
if status.hard_limit and not existing then return nil, "hard_limit" end if status.hard_limit and not existing then return nil, "hard_limit" end
@@ -421,7 +425,6 @@ function Outbox:enqueue(snapshot, opts)
local stored_snapshot = {} local stored_snapshot = {}
for key, value in pairs(snapshot) do stored_snapshot[key] = value end for key, value in pairs(snapshot) do stored_snapshot[key] = value end
stored_snapshot.stats_ids = copyArray(snapshot.stats_ids) stored_snapshot.stats_ids = copyArray(snapshot.stats_ids)
local existing_with_payload = existing and self:attachPayload(existing) or nil
if existing_with_payload and existing_with_payload.acknowledged.stats ~= true then if existing_with_payload and existing_with_payload.acknowledged.stats ~= true then
stored_snapshot.stats_ids = mergeUniqueArrays( stored_snapshot.stats_ids = mergeUniqueArrays(
existing_with_payload.snapshot.stats_ids, stored_snapshot.stats_ids) existing_with_payload.snapshot.stats_ids, stored_snapshot.stats_ids)
@@ -42,6 +42,7 @@ function BookOrbitState.snapshot(tables, flush_handler)
books = deepCopy(tables.books or {}), books = deepCopy(tables.books or {}),
unmatched = deepCopy(tables.unmatched or {}), unmatched = deepCopy(tables.unmatched or {}),
files = deepCopy(tables.files or {}), files = deepCopy(tables.files or {}),
statsRows = deepCopy(tables.statsRows or {}),
global = deepCopy(tables.global or {}), global = deepCopy(tables.global or {}),
flush_handler = flush_handler, flush_handler = flush_handler,
}, BookOrbitState) }, BookOrbitState)
@@ -57,6 +58,7 @@ function BookOrbitState:reload()
self.books = settings:readSetting("books", {}) self.books = settings:readSetting("books", {})
self.unmatched = settings:readSetting("unmatched", {}) self.unmatched = settings:readSetting("unmatched", {})
self.files = settings:readSetting("files", {}) self.files = settings:readSetting("files", {})
self.statsRows = settings:readSetting("statsRows", {})
self.global = settings:readSetting("global", {}) self.global = settings:readSetting("global", {})
return self return self
end end
@@ -65,6 +67,40 @@ function BookOrbitState:getBook(md5)
return self.books[md5] return self.books[md5]
end end
-- A partial MD5 may identify several KOReader statistics rows. Those rows
-- need independent cursors: advancing one book's watermark must never hide
-- an older event belonging to another book with the same digest. Row ids are
-- local to statistics.sqlite3, so the persisted metadata guards against an id
-- being reused after that database is rebuilt.
function BookOrbitState:getStatsRow(id, md5, title, authors)
id = tonumber(id)
if not id or not md5 then return nil end
local key = tostring(id)
local row = self.statsRows[key]
if not row or row.md5 ~= md5 or row.title ~= title or row.authors ~= authors then
row = {
md5 = md5,
title = title,
authors = authors,
statsWatermark = 0,
}
self.statsRows[key] = row
end
row.statsWatermark = row.statsWatermark or 0
return row
end
function BookOrbitState:bindStatsRow(id, md5, title, authors, book_file_id, book_id)
local row = self:getStatsRow(id, md5, title, authors)
if not row then return nil end
if row.bookFileId ~= book_file_id or row.bookId ~= book_id then
row.statsWatermark = 0
end
row.bookFileId = book_file_id
row.bookId = book_id
return row
end
-- Maps BookOrbit bookId -> local file path and bookFileId -> local file path -- Maps BookOrbit bookId -> local file path and bookFileId -> local file path
-- for every matched book whose file is still present on disk. Both maps come -- for every matched book whose file is still present on disk. Both maps come
-- from one scan with one existence check per stored file; bookorbit_state_manager -- from one scan with one existence check per stored file; bookorbit_state_manager
@@ -123,11 +123,12 @@ local function scopeTables(state, scope)
books = state.books, books = state.books,
unmatched = state.unmatched, unmatched = state.unmatched,
files = state.files, files = state.files,
statsRows = state.statsRows,
global = state.global, global = state.global,
} }
end end
local tables = { books = {}, unmatched = {}, files = {}, global = {} } local tables = { books = {}, unmatched = {}, files = {}, statsRows = {}, global = {} }
for digest in pairs(scope.digests) do for digest in pairs(scope.digests) do
tables.books[digest] = state.books[digest] tables.books[digest] = state.books[digest]
tables.unmatched[digest] = state.unmatched[digest] tables.unmatched[digest] = state.unmatched[digest]
@@ -135,21 +136,28 @@ local function scopeTables(state, scope)
for file in pairs(scope.files) do for file in pairs(scope.files) do
tables.files[file] = state.files[file] tables.files[file] = state.files[file]
end end
for row_id in pairs(scope.statsRows) do
tables.statsRows[row_id] = state.statsRows[row_id]
end
if scope.global then tables.global = state.global end if scope.global then tables.global = state.global end
return tables return tables
end end
local function normalizeScope(opts) local function normalizeScope(opts)
if not opts or opts.full then if not opts or opts.full then
return { full = true, digests = {}, files = {}, global = true } return { full = true, digests = {}, files = {}, statsRows = {}, global = true }
end end
local scope = { full = false, digests = {}, files = {}, global = opts.global ~= false } local scope = { full = false, digests = {}, files = {}, statsRows = {}, global = opts.global ~= false }
for _, digest in ipairs(opts.digests or {}) do for _, digest in ipairs(opts.digests or {}) do
if digest then scope.digests[digest] = true end if digest then scope.digests[digest] = true end
end end
for _, file in ipairs(opts.files or {}) do for _, file in ipairs(opts.files or {}) do
if file then scope.files[file] = true end if file then scope.files[file] = true end
end end
for _, row_id in ipairs(opts.statsRows or {}) do
row_id = tonumber(row_id)
if row_id then scope.statsRows[tostring(row_id)] = true end
end
return scope return scope
end end
@@ -158,6 +166,7 @@ local function refreshSession(session)
session.books = deepCopy(current.books) session.books = deepCopy(current.books)
session.unmatched = deepCopy(current.unmatched) session.unmatched = deepCopy(current.unmatched)
session.files = deepCopy(current.files) session.files = deepCopy(current.files)
session.statsRows = deepCopy(current.statsRows)
session.global = deepCopy(current.global) session.global = deepCopy(current.global)
session.manager_base = deepCopy(current) session.manager_base = deepCopy(current)
session.manager_generation = generation session.manager_generation = generation
@@ -172,6 +181,7 @@ local function commitSession(session)
replaceTable(state.books, mergeValue(base.books, session.books, state.books)) replaceTable(state.books, mergeValue(base.books, session.books, state.books))
replaceTable(state.unmatched, mergeValue(base.unmatched, session.unmatched, state.unmatched)) replaceTable(state.unmatched, mergeValue(base.unmatched, session.unmatched, state.unmatched))
replaceTable(state.files, mergeValue(base.files, session.files, state.files)) replaceTable(state.files, mergeValue(base.files, session.files, state.files))
replaceTable(state.statsRows, mergeValue(base.statsRows, session.statsRows, state.statsRows))
replaceTable(state.global, mergeValue(base.global, session.global, state.global)) replaceTable(state.global, mergeValue(base.global, session.global, state.global))
else else
for digest in pairs(scope.digests) do for digest in pairs(scope.digests) do
@@ -182,6 +192,9 @@ local function commitSession(session)
for file in pairs(scope.files) do for file in pairs(scope.files) do
state.files[file] = mergeValue(base.files[file], session.files[file], state.files[file]) state.files[file] = mergeValue(base.files[file], session.files[file], state.files[file])
end end
for row_id in pairs(scope.statsRows) do
state.statsRows[row_id] = mergeValue(base.statsRows[row_id], session.statsRows[row_id], state.statsRows[row_id])
end
if scope.global then if scope.global then
replaceTable(state.global, mergeValue(base.global, session.global, state.global)) replaceTable(state.global, mergeValue(base.global, session.global, state.global))
end end
@@ -38,6 +38,12 @@ local function applyRow(entry, id, title, authors, last_open)
if title == "" then title = nil end if title == "" then title = nil end
if authors == "" then authors = nil end if authors == "" then authors = nil end
table.insert(entry.ids, id) table.insert(entry.ids, id)
table.insert(entry.rows, {
id = id,
title = title,
authors = authors,
last_open = last_open,
})
if not entry._variant_seen[metadataKey(title, authors)] then if not entry._variant_seen[metadataKey(title, authors)] then
entry._variant_seen[metadataKey(title, authors)] = true entry._variant_seen[metadataKey(title, authors)] = true
entry._variant_count = entry._variant_count + 1 entry._variant_count = entry._variant_count + 1
@@ -53,6 +59,7 @@ end
local function finalizeEntry(entry) local function finalizeEntry(entry)
entry.metadata_ambiguous = (entry._variant_count or 0) > 1 entry.metadata_ambiguous = (entry._variant_count or 0) > 1
entry.stats_ambiguous = #entry.ids > 1
entry._variant_seen = nil entry._variant_seen = nil
entry._variant_count = nil entry._variant_count = nil
return entry return entry
@@ -163,7 +170,7 @@ function Session:book(md5)
if type(md5) ~= "string" or not md5:match("^%x+$") then return nil end if type(md5) ~= "string" or not md5:match("^%x+$") then return nil end
local res = self:select(BOOK_BY_MD5_SQL, { md5 }) local res = self:select(BOOK_BY_MD5_SQL, { md5 })
if not res then return nil end if not res then return nil end
local book = { ids = {}, last_open = 0, _variant_seen = {}, _variant_count = 0 } local book = { ids = {}, rows = {}, last_open = 0, _variant_seen = {}, _variant_count = 0 }
for i = 1, #res[1] do for i = 1, #res[1] do
applyRow(book, tonumber(res[1][i]), res[2][i], res[3][i], tonumber(res[4][i]) or 0) applyRow(book, tonumber(res[1][i]), res[2][i], res[3][i], tonumber(res[4][i]) or 0)
end end
@@ -318,7 +325,7 @@ end
function BookOrbitStatsReader.collectBookRow(collector, row) function BookOrbitStatsReader.collectBookRow(collector, row)
local entry = collector.by_md5[row.md5] local entry = collector.by_md5[row.md5]
if not entry then if not entry then
entry = { md5 = row.md5, ids = {}, last_open = 0, _variant_seen = {}, _variant_count = 0 } entry = { md5 = row.md5, ids = {}, rows = {}, last_open = 0, _variant_seen = {}, _variant_count = 0 }
collector.by_md5[row.md5] = entry collector.by_md5[row.md5] = entry
table.insert(collector.list, entry) table.insert(collector.list, entry)
end end
@@ -334,7 +341,7 @@ function BookOrbitStatsReader.finalizeBookEntry(entry)
return finalizeEntry(entry) return finalizeEntry(entry)
end end
-- Returns { ids, last_open, title, authors, metadata_ambiguous } for one md5. -- Returns { ids, rows, last_open, title, authors, metadata_ambiguous, stats_ambiguous } for one md5.
-- Used by live single-book sync before match-check. -- Used by live single-book sync before match-check.
function BookOrbitStatsReader.getBook(md5) function BookOrbitStatsReader.getBook(md5)
return withSession(function(session) return withSession(function(session)
@@ -163,9 +163,10 @@ end
local function chunkSummary(ctx) local function chunkSummary(ctx)
return string.format( return string.format(
"enumerateMs=%.0f partialMd5Ms=%.0f partialMd5Count=%d sidecarMs=%.0f slowestItemMs=%.0f statsSkipped=%d", "enumerateMs=%.0f partialMd5Ms=%.0f partialMd5Count=%d sidecarMs=%.0f slowestItemMs=%.0f statsSkipped=%d ambiguousStats=%d",
ctx.timing.enumerate_ms, ctx.timing.partial_md5_ms, ctx.timing.partial_md5_count, ctx.timing.enumerate_ms, ctx.timing.partial_md5_ms, ctx.timing.partial_md5_count,
ctx.timing.sidecar_ms, ctx.timing.max_item_ms, ctx.counts.stats_skipped or 0) ctx.timing.sidecar_ms, ctx.timing.max_item_ms, ctx.counts.stats_skipped or 0,
ctx.counts.stats_ambiguous or 0)
end end
local function finish(ctx, err) local function finish(ctx, err)
@@ -204,10 +205,15 @@ local function finish(ctx, err)
elseif not err and ctx.had_errors then elseif not err and ctx.had_errors then
ctx.plugin:recordSyncError("sweep", "partial_failure") ctx.plugin:recordSyncError("sweep", "partial_failure")
elseif not err and ctx.plugin.recordSyncSuccess then elseif not err and ctx.plugin.recordSyncSuccess then
local message = T(_("%1 books, %2 reading events, %3 highlights"),
ctx.counts.books_matched, ctx.counts.page_stats, ctx.counts.annotations)
if (ctx.counts.stats_ambiguous or 0) > 0 then
message = message .. ". " .. T(_("%1 books need to be opened and synced once to recover their reading history."),
ctx.counts.stats_ambiguous)
end
ctx.plugin:recordSyncSuccess( ctx.plugin:recordSyncSuccess(
"sweep", "sweep",
T(_("%1 books, %2 reading events, %3 highlights"), message)
ctx.counts.books_matched, ctx.counts.page_stats, ctx.counts.annotations))
end end
end end
@@ -270,6 +276,10 @@ local function finish(ctx, err)
if ctx.had_errors then if ctx.had_errors then
text = text .. "\n" .. _("Some books failed and will retry on the next sync.") text = text .. "\n" .. _("Some books failed and will retry on the next sync.")
end end
if (ctx.counts.stats_ambiguous or 0) > 0 then
text = text .. "\n" .. T(_("%1 books need to be opened and synced once to recover their reading history."),
ctx.counts.stats_ambiguous)
end
UIManager:show(InfoMessage:new{ text = text, timeout = 5 }) UIManager:show(InfoMessage:new{ text = text, timeout = 5 })
end end
logger.info(string.format( logger.info(string.format(
@@ -421,12 +431,13 @@ local function stepEnumerateCandidates(ctx)
BookOrbitStatsReader.finalizeBookEntry(entry) BookOrbitStatsReader.finalizeBookEntry(entry)
chunk_ctx.candidates[entry.md5] = { chunk_ctx.candidates[entry.md5] = {
stat_ids = entry.ids, stat_ids = entry.ids,
stats_rows = entry.rows,
last_open = entry.last_open, last_open = entry.last_open,
title = entry.title, title = entry.title,
authors = entry.authors, authors = entry.authors,
source = "statistics", source = "statistics",
metadata_ambiguous = entry.metadata_ambiguous, metadata_ambiguous = entry.metadata_ambiguous,
stats_metadata_ambiguous = entry.metadata_ambiguous, stats_metadata_ambiguous = entry.stats_ambiguous,
} }
end end
@@ -600,11 +611,30 @@ local function buildStatsQueue(ctx)
local latest = ctx.stats_session and ctx.stats_session:latestEventTimes() or nil local latest = ctx.stats_session and ctx.stats_session:latestEventTimes() or nil
ctx.stats_queue = BookOrbitQueue.new() ctx.stats_queue = BookOrbitQueue.new()
ctx.counts.stats_skipped = 0 ctx.counts.stats_skipped = 0
ctx.counts.stats_ambiguous = 0
for md5, cand in pairs(ctx.candidates) do for md5, cand in pairs(ctx.candidates) do
if cand.stat_ids and not cand.stats_metadata_ambiguous then local book = ctx.state:getBook(md5)
local book = ctx.state:getBook(md5) if cand.stat_ids and book then
if book then if cand.stats_metadata_ambiguous then
for _, row in ipairs(cand.stats_rows or {}) do
local target = ctx.state:getStatsRow(row.id, md5, row.title, row.authors)
if target.bookFileId then
if latest and not hasNewEvents(latest, { row.id }, target.statsWatermark or 0) then
ctx.counts.stats_skipped = ctx.counts.stats_skipped + 1
else
ctx.stats_queue:push({
md5 = md5,
ids = { row.id },
target = target,
book_file_id = target.bookFileId,
})
end
elseif not latest or (latest[row.id] or 0) > (target.statsWatermark or 0) then
ctx.counts.stats_ambiguous = ctx.counts.stats_ambiguous + 1
end
end
else
if latest and not hasNewEvents(latest, cand.stat_ids, book.statsWatermark or 0) then if latest and not hasNewEvents(latest, cand.stat_ids, book.statsWatermark or 0) then
ctx.counts.stats_skipped = ctx.counts.stats_skipped + 1 ctx.counts.stats_skipped = ctx.counts.stats_skipped + 1
else else
@@ -681,7 +711,8 @@ local function stepStatsNext(ctx)
return return
end end
local watermark = book.statsWatermark or 0 local stats_target = item.target or book
local watermark = stats_target.statsWatermark or 0
local events = ctx.stats_session local events = ctx.stats_session
and ctx.stats_session:eventsAfter(item.ids, watermark, STATS_BATCH) and ctx.stats_session:eventsAfter(item.ids, watermark, STATS_BATCH)
or BookOrbitStatsReader.getEventsAfter(item.ids, watermark, STATS_BATCH) or BookOrbitStatsReader.getEventsAfter(item.ids, watermark, STATS_BATCH)
@@ -690,7 +721,9 @@ local function stepStatsNext(ctx)
return return
end end
local body, err = ctx.client:uploadPageStats({ { hash = item.md5, events = events } }) local upload = { hash = item.md5, events = events }
if item.book_file_id then upload.bookFileId = item.book_file_id end
local body, err = ctx.client:uploadPageStats({ upload })
if aborted(ctx) then return end if aborted(ctx) then return end
if not body then if not body then
if isAuthError(err) then return finish(ctx, "auth") end if isAuthError(err) then return finish(ctx, "auth") end
@@ -702,13 +735,20 @@ local function stepStatsNext(ctx)
for _, unmatched in ipairs(body.unmatched or {}) do for _, unmatched in ipairs(body.unmatched or {}) do
if unmatched == item.md5 then if unmatched == item.md5 then
ctx.state:setUnmatched(item.md5) if item.book_file_id then
stats_target.bookFileId = nil
stats_target.bookId = nil
stats_target.statsWatermark = 0
ctx.had_errors = true
else
ctx.state:setUnmatched(item.md5)
end
step(ctx, ctx.steps.statsNext) step(ctx, ctx.steps.statsNext)
return return
end end
end end
local more = BookOrbitState.applyStatsAck(book, events, body, item.md5, STATS_BATCH, watermark) local more = BookOrbitState.applyStatsAck(stats_target, events, body, item.md5, STATS_BATCH, watermark)
ctx.counts.page_stats = ctx.counts.page_stats + #events ctx.counts.page_stats = ctx.counts.page_stats + #events
if more then if more then
ctx.current_stats = item ctx.current_stats = item
+1 -1
View File
@@ -48,7 +48,7 @@ local BookOrbitProgressSync = require("bookorbit_progress_sync")
local BookOrbitSweep = require("bookorbit_sweep") local BookOrbitSweep = require("bookorbit_sweep")
local BookOrbitUpdater = require("bookorbit_updater") local BookOrbitUpdater = require("bookorbit_updater")
local PLUGIN_VERSION = "1.5.3" local PLUGIN_VERSION = "1.5.4"
local SYNC_STRATEGY = { local SYNC_STRATEGY = {
PROMPT = 1, PROMPT = 1,
@@ -11,6 +11,9 @@ local calls
local last_match_candidates local last_match_candidates
local stats_watermarks local stats_watermarks
local page_stat_events local page_stat_events
local last_stats_upload
local bound_stats_row
local stored_stats_row
local match_fresh = false local match_fresh = false
local exchange_skippable = false local exchange_skippable = false
@@ -55,9 +58,10 @@ package.loaded["bookorbit_api"] = {
matchCheck = function(_, _, candidates) matchCheck = function(_, _, candidates)
table.insert(calls, "match") table.insert(calls, "match")
last_match_candidates = candidates last_match_candidates = candidates
local target = candidates.abcdef.book_file_id or 2
return { return {
libraryVersion = "lib-v1", libraryVersion = "lib-v1",
matches = { { hash = "abcdef", bookFileId = 2, bookId = 1 } }, matches = { { hash = "abcdef", bookFileId = target, bookId = target + 10 } },
} }
end, end,
uploadBookStates = function() uploadBookStates = function()
@@ -66,6 +70,7 @@ package.loaded["bookorbit_api"] = {
end, end,
uploadPageStats = function(_, books) uploadPageStats = function(_, books)
table.insert(calls, "stats") table.insert(calls, "stats")
last_stats_upload = books[1]
return { return {
results = { { hash = books[1].hash, watermark = books[1].events[#books[1].events].startTime } }, results = { { hash = books[1].hash, watermark = books[1].events[#books[1].events].startTime } },
unmatched = {}, unmatched = {},
@@ -103,9 +108,29 @@ local book
local state = { local state = {
global = { libraryVersion = "lib-v1" }, global = { libraryVersion = "lib-v1" },
getBook = function() return book end, getBook = function() return book end,
setMatched = function() end, setMatched = function(_, _, book_file_id, book_id, file)
book.fileId = book_file_id
book.bookId = book_id
book.file = file or book.file
end,
setUnmatched = function() book = nil end, setUnmatched = function() book = nil end,
rememberFile = function() end, rememberFile = function() end,
getStatsRow = function(_, id, md5, title, authors)
if not stored_stats_row then
stored_stats_row = {
id = id, md5 = md5, title = title, authors = authors, statsWatermark = 0,
}
end
return stored_stats_row
end,
bindStatsRow = function(_, id, md5, title, authors, book_file_id, book_id)
bound_stats_row = {
id = id, md5 = md5, title = title, authors = authors,
bookFileId = book_file_id, bookId = book_id, statsWatermark = 0,
}
stored_stats_row = bound_stats_row
return bound_stats_row
end,
flush = function() end, flush = function() end,
} }
package.loaded["bookorbit_state_manager"] = { session = function() return state end } package.loaded["bookorbit_state_manager"] = { session = function() return state end }
@@ -137,6 +162,9 @@ local function run(opts)
calls = {} calls = {}
stats_watermarks = {} stats_watermarks = {}
page_stat_events = opts.page_stat_events or {} page_stat_events = opts.page_stat_events or {}
last_stats_upload = nil
bound_stats_row = nil
stored_stats_row = opts.stored_stats_row
book = { book = {
bookId = 1, bookId = 1,
fileId = 2, fileId = 2,
@@ -153,6 +181,8 @@ local function run(opts)
file = "/books/a.epub", file = "/books/a.epub",
stats_ids = { 42 }, stats_ids = { 42 },
stats_identity_repaired = opts.stats_identity_repaired == true, stats_identity_repaired = opts.stats_identity_repaired == true,
stats_row_ambiguous = opts.stats_row_ambiguous == true,
stats_row = opts.stats_row,
annotations = {}, annotations = {},
ann_count = 0, ann_count = 0,
ann_signature = "0::0:0", ann_signature = "0::0:0",
@@ -223,4 +253,36 @@ requests = run{
assertEqual(stats_watermarks[1], 0, "identity repair replays exact-row history before the stored watermark") assertEqual(stats_watermarks[1], 0, "identity repair replays exact-row history before the stored watermark")
assertEqual(requests, "stats,progress", "the stranded historical event is uploaded under the repaired digest") assertEqual(requests, "stats,progress", "the stranded historical event is uploaded under the repaired digest")
match_fresh = true
requests = run{
stats_row_ambiguous = true,
stats_row = { id = 42, title = "Collision Two", authors = "Author B" },
stats_watermark = 900,
page_stat_events = { { page = 2, startTime = 600, durationSeconds = 60, totalPages = 100 } },
}
assertEqual(bound_stats_row.id, 42, "the exact live KOReader statistics row is bound")
assertEqual(bound_stats_row.bookFileId, 2, "the row is bound to the verified server file")
assertEqual(stats_watermarks[1], 0, "a newly bound collision row replays its own complete history")
assertEqual(last_stats_upload.bookFileId, 2, "collision events carry the explicit verified server target")
assertEqual(last_match_candidates.abcdef.book_file_id, nil,
"a collision match never proposes the sibling hash-level file target")
assertEqual(requests, "match,stats,progress",
"a collision row revalidates its target instead of trusting the shared hash cache")
requests = run{
stats_row_ambiguous = true,
stats_row = { id = 42, title = "Collision Two", authors = "Author B" },
stored_stats_row = {
id = 42,
md5 = "abcdef",
title = "Collision Two",
authors = "Author B",
bookFileId = 7,
bookId = 17,
statsWatermark = 500,
},
}
assertEqual(last_match_candidates.abcdef.book_file_id, 7,
"an established row-specific target is reused for collision revalidation")
print("bookorbit_book_sync_fast_path_test.lua: ok") print("bookorbit_book_sync_fast_path_test.lua: ok")
@@ -123,6 +123,8 @@ assertEqual(sqlite.opens, 1, "capture opens no database when identity is primed"
assertEqual(flushes, 1, "the pending page-stat flush still runs live at capture") assertEqual(flushes, 1, "the pending page-stat flush still runs live at capture")
assertEqual(#snap.stats_ids, 1, "the snapshot carries the statistics row ids") assertEqual(#snap.stats_ids, 1, "the snapshot carries the statistics row ids")
assertEqual(snap.stats_ids[1], 42, "the row id comes from the primed identity") assertEqual(snap.stats_ids[1], 42, "the row id comes from the primed identity")
assertEqual(snap.stats_row.id, 42, "the exact live statistics row is carried with its metadata")
assertEqual(snap.stats_row_ambiguous, false, "a single statistics row needs no explicit collision routing")
assertEqual(snap.title, "Statistics Title", "stable identity fields come from the cache") assertEqual(snap.title, "Statistics Title", "stable identity fields come from the cache")
assertEqual(snap.percentage, 0.5, "progress is read live, never cached") assertEqual(snap.percentage, 0.5, "progress is read live, never cached")
assert(type(snap.ann_signature) == "string" and snap.ann_signature ~= "", assert(type(snap.ann_signature) == "string" and snap.ann_signature ~= "",
@@ -145,6 +147,22 @@ local warm = BookSync.capture(plugin)
assertEqual(sqlite.opens, 1, "the fallback primes the cache for the next handler") assertEqual(sqlite.opens, 1, "the fallback primes the cache for the next handler")
assertEqual(warm.stats_ids[1], 42, "the warmed capture carries the same row ids") assertEqual(warm.stats_ids[1], 42, "the warmed capture carries the same row ids")
-- A genuine partial-MD5 collision still has an exact live KOReader row while
-- the book is open. Capture must preserve that row rather than the whole hash
-- group so the drain can bind and replay it safely.
book_rows = {
{ "42", "Collision One", "Author A", "1700000000" },
{ "43", "Collision Two", "Author B", "1700000100" },
}
plugin.ui.statistics.id_curr_book = 43
BookOrbitStatsReader.forgetIdentity()
local collision = BookSync.capture(plugin)
assertEqual(collision.stats_row_ambiguous, true, "the exact row records that its digest collides")
assertEqual(collision.stats_row.id, 43, "capture selects the live collision row")
assertEqual(collision.stats_row.title, "Collision Two", "the selected row keeps its identity guard")
assertEqual(#collision.stats_ids, 1, "only the live collision row is eligible for this book sync")
assertEqual(collision.stats_ids[1], 43, "the other book's statistics row is never included")
-- A repaired digest can be a cached miss while KOReader still has historical -- A repaired digest can be a cached miss while KOReader still has historical
-- events attached to the current row's old digest. The live row id is exact, -- events attached to the current row's old digest. The live row id is exact,
-- so capture carries it under the recomputed file hash without a second query. -- so capture carries it under the recomputed file hash without a second query.
@@ -65,6 +65,15 @@ assertEqual(replacement.snapshot.stats_ids[3], 29, "the newest statistics row is
assertEqual(replacement.snapshot.stats_identity_repaired, true, assertEqual(replacement.snapshot.stats_identity_repaired, true,
"a pending identity-repair replay survives snapshot replacement") "a pending identity-repair replay survives snapshot replacement")
local collision_snapshot = {}
for key, value in pairs(snapshot) do collision_snapshot[key] = value end
collision_snapshot.file = "/books/collision.epub"
collision_snapshot.stats_ids = { 41 }
local collision = assert(outbox:enqueue(collision_snapshot, { reason = "close", annotation_sync = true }))
assert(collision.id ~= replacement.id, "colliding files keep independent outbox entries")
assertEqual(outbox:status().count, 2, "a shared digest cannot coalesce distinct files")
outbox:removeEntry(collision.id)
fixture.fail_write = true fixture.fail_write = true
local failed, failure = outbox:enqueue(snapshot, { reason = "close" }) local failed, failure = outbox:enqueue(snapshot, { reason = "close" })
fixture.fail_write = false fixture.fail_write = false
@@ -164,6 +164,26 @@ assert(unscoped:getBook("legacy-cursor").annWatermark == "",
replaced:setUnmatched("moved") replaced:setUnmatched("moved")
assert(replaced:getBook("moved") == nil, "an unmatched response removes the complete matched record") assert(replaced:getBook("moved") == nil, "an unmatched response removes the complete matched record")
-- Partial-MD5 collisions use one guarded cursor per KOReader statistics row.
-- Rebinding a row to a different server file replays it, while another row
-- sharing the same digest remains independent.
local collision = newState({ libraryVersion = "lib-v1" })
local first_row = collision:bindStatsRow(41, "shared", "First", "Author A", 11, 1)
first_row.statsWatermark = 500
local second_row = collision:bindStatsRow(42, "shared", "Second", "Author B", 22, 2)
second_row.statsWatermark = 600
assert(collision:getStatsRow(41, "shared", "First", "Author A").statsWatermark == 500,
"the first collision row keeps its own watermark")
assert(collision:getStatsRow(42, "shared", "Second", "Author B").statsWatermark == 600,
"the second collision row keeps its own watermark")
collision:bindStatsRow(41, "shared", "First", "Author A", 33, 3)
assert(collision:getStatsRow(41, "shared", "First", "Author A").statsWatermark == 0,
"a changed explicit target replays that row")
assert(collision:getStatsRow(42, "shared", "Second", "Author B").statsWatermark == 600,
"rebinding one collision row does not reset another")
assert(collision:getStatsRow(42, "shared", "Second renamed", "Author B").bookFileId == nil,
"changed row metadata invalidates a potentially reused local row id")
-- State written before the stamp existed must recheck once, not be trusted. -- State written before the stamp existed must recheck once, not be trusted.
local legacy = newState({ libraryVersion = "lib-v1" }) local legacy = newState({ libraryVersion = "lib-v1" })
legacy.books["legacy1"] = { bookId = 3, fileId = 4, file = "/books/legacy.epub" } legacy.books["legacy1"] = { bookId = 3, fileId = 4, file = "/books/legacy.epub" }
@@ -45,6 +45,46 @@ do
assertEqual(stats.queries, 0, "and never reaches the database") assertEqual(stats.queries, 0, "and never reaches the database")
end end
-- Collision groups retain each row's metadata, not just the combined id list,
-- so callers can persist a guarded per-row server identity and cursor.
do
local Reader = load(function(sql)
if sql:find("FROM book WHERE md5", 1, true) then
return FakeSqlite.resultSet{
{ "7", "Collision One", "Author A", "100" },
{ "9", "Collision Two", "Author B", "200" },
}
end
return nil
end)
local book = Reader.getBook("abc123")
assertEqual(book.metadata_ambiguous, true, "different metadata marks the digest ambiguous")
assertEqual(book.stats_ambiguous, true, "multiple rows mark statistics routing ambiguous")
assertEqual(#book.rows, 2, "both guarded row identities are retained")
assertEqual(book.rows[1].id, 7, "the first row id is numeric")
assertEqual(book.rows[2].title, "Collision Two", "row metadata stays attached to its id")
assertEqual(book.rows[2].last_open, 200, "row activity remains available independently")
end
-- SQLite UNIQUE constraints permit repeated NULL values, so the collision
-- guard must count rows rather than relying on distinct metadata variants.
do
local Reader = load(function(sql)
if sql:find("FROM book WHERE md5", 1, true) then
return FakeSqlite.resultSet{
{ "7", "Same Title", nil, "100" },
{ "9", "Same Title", nil, "200" },
}
end
return nil
end)
local book = Reader.getBook("abc123")
assertEqual(book.metadata_ambiguous, false, "identical metadata remains usable for matching")
assertEqual(book.stats_ambiguous, true, "two rows still require independent statistics routing")
end
-- Event batching binds the row ids, then the watermark, then the limit. -- Event batching binds the row ids, then the watermark, then the limit.
do do
local seen local seen
@@ -288,6 +288,101 @@ do
"sweep recovery carries only the actual digest's verified file id") "sweep recovery carries only the actual digest's verified file id")
end end
-- Statistics rows that share a partial MD5 keep independent server targets
-- and watermarks. The older cursor for one book must not hide newer reading
-- from another book in the collision group.
do
local hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
local harness = SweepHarness.install{
books = {
{ md5 = hash, id = 41, title = "Collision One", authors = "Author A", last_open = 500 },
{ md5 = hash, id = 42, title = "Collision Two", authors = "Author B", last_open = 600 },
},
events = {
[41] = { { page = 1, start_time = 500 } },
[42] = { { page = 2, start_time = 600 } },
},
library_version = "v1",
state = {
books = {
[hash] = {
bookId = 1, fileId = 11, statsWatermark = 900,
matchVerifiedAt = NOW, matchVerifiedVersion = "v1",
},
},
statsRows = {
["41"] = {
md5 = hash, title = "Collision One", authors = "Author A",
bookId = 1, bookFileId = 11, statsWatermark = 550,
},
["42"] = {
md5 = hash, title = "Collision Two", authors = "Author B",
bookId = 2, bookFileId = 22, statsWatermark = 0,
},
},
global = { libraryVersion = "v1" },
},
page_stats_response = function(books)
return {
results = { { hash = books[1].hash, watermark = books[1].events[1].startTime } },
unmatched = {},
}
end,
}
local Sweep = require("bookorbit_sweep")
startSweep(harness, Sweep)
harness.scheduler:drain()
assertEqual(#harness.calls.page_stats, 1, "only the collision row with new events uploads")
assertEqual(harness.calls.page_stats[1][1].bookFileId, 22,
"the collision row carries its independently verified server file id")
assertEqual(harness.calls.page_stats[1][1].events[1].startTime, 600,
"the other collision row's newer watermark does not suppress this event")
assertEqual(harness.state.statsRows["41"].statsWatermark, 550,
"the unchanged collision row keeps its own watermark")
assertEqual(harness.state.statsRows["42"].statsWatermark, 600,
"the uploaded collision row advances only its own watermark")
end
-- Never guess a server target for a collision row. It stays pending and the
-- successful sweep tells the user exactly how to establish the missing link.
do
local hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
local harness = SweepHarness.install{
books = {
{ md5 = hash, id = 51, title = "Known", authors = "Author A", last_open = 500 },
{ md5 = hash, id = 52, title = "Needs Link", authors = "Author B", last_open = 600 },
},
events = { [52] = { { page = 2, start_time = 600 } } },
library_version = "v1",
state = {
books = {
[hash] = {
bookId = 1, fileId = 11, statsWatermark = 0,
matchVerifiedAt = NOW, matchVerifiedVersion = "v1",
},
},
statsRows = {
["51"] = {
md5 = hash, title = "Known", authors = "Author A",
bookId = 1, bookFileId = 11, statsWatermark = 0,
},
},
global = { libraryVersion = "v1" },
},
}
local Sweep = require("bookorbit_sweep")
startSweep(harness, Sweep)
harness.scheduler:drain()
assertEqual(#harness.calls.page_stats, 0, "an unbound collision row is never attributed by guesswork")
assert(harness.shown.text:find("1 books need to be opened and synced once", 1, true),
"the completed sweep exposes the recovery action")
assertEqual(harness.calls.sweep_complete, 1, "the diagnostic does not turn safe deferral into a failed sweep")
end
-- Cancellation stops the run at its next yield: the step already scheduled -- Cancellation stops the run at its next yield: the step already scheduled
-- runs, sees it is no longer the current generation and writes nothing. -- runs, sees it is no longer the current generation and writes nothing.
do do
@@ -127,7 +127,7 @@ opts:
- books: statistics rows, { md5, id, title, authors, last_open } - books: statistics rows, { md5, id, title, authors, last_open }
- events: map of statistics row id to { page, start_time, duration, total_pages } - events: map of statistics row id to { page, start_time, duration, total_pages }
- history: ReadHistory entries, { file, time, text } - history: ReadHistory entries, { file, time, text }
- state: initial { books, unmatched, files, global } -- state: initial { books, unmatched, files, statsRows, global }
- library_version: token every server response carries - library_version: token every server response carries
- unmatched: set of hashes match-check refuses to match - unmatched: set of hashes match-check refuses to match
- matches: map of hashes to custom { bookId, bookFileId } match results - matches: map of hashes to custom { bookId, bookFileId } match results
@@ -299,6 +299,7 @@ function SweepHarness.install(opts)
books = initial.books or {}, books = initial.books or {},
unmatched = initial.unmatched or {}, unmatched = initial.unmatched or {},
files = initial.files or {}, files = initial.files or {},
statsRows = initial.statsRows or {},
global = initial.global or {}, global = initial.global or {},
}, function() }, function()
handle.flushes = handle.flushes + 1 handle.flushes = handle.flushes + 1
@@ -119,6 +119,11 @@ export class PageStatsBookDto {
@Matches(MD5_HEX) @Matches(MD5_HEX)
hash!: string; hash!: string;
@IsOptional()
@IsInt()
@Min(1)
bookFileId?: number;
@IsArray() @IsArray()
@ArrayMinSize(1) @ArrayMinSize(1)
@ValidateNested({ each: true }) @ValidateNested({ each: true })
@@ -321,22 +321,26 @@ describe('KOReader plugin update source wiring', () => {
expect(statsReader).toContain('function BookOrbitStatsReader.primeIdentity(md5)'); expect(statsReader).toContain('function BookOrbitStatsReader.primeIdentity(md5)');
expect(statsReader).toContain('local book = BookOrbitStatsReader.getBook(md5)'); expect(statsReader).toContain('local book = BookOrbitStatsReader.getBook(md5)');
expect(bookSync).toContain('local current_stats_id = stats and tonumber(stats.id_curr_book) or nil'); expect(bookSync).toContain('local current_stats_id = stats and tonumber(stats.id_curr_book) or nil');
expect(bookSync).toContain('local stats_ambiguous = current_stats_id == nil and metadata.metadata_ambiguous == true'); expect(bookSync).toContain('local stats_ambiguous = current_stats_id == nil and metadata.stats_ambiguous == true');
expect(bookSync).toContain('table.insert(stats_ids, current_stats_id)'); expect(bookSync).toContain('table.insert(stats_ids, current_stats_id)');
expect(bookSync).toContain('title = stats_ambiguous and titleFromFile(file) or (metadata.title or titleFromFile(file))'); expect(bookSync).toContain('title = stats_ambiguous and titleFromFile(file) or (metadata.title or titleFromFile(file))');
expect(bookSync).toContain('authors = stats_ambiguous and nil or metadata.authors'); expect(bookSync).toContain('authors = stats_ambiguous and nil or metadata.authors');
expect(bookSync).toContain('last_open = metadata.last_open or ts'); expect(bookSync).toContain('last_open = metadata.last_open or ts');
expect(bookSync).toContain('metadata_ambiguous = false'); expect(bookSync).toContain('metadata_ambiguous = false');
expect(bookSync).toContain('stats_metadata_ambiguous = stats_ambiguous'); expect(bookSync).toContain('stats_metadata_ambiguous = stats_ambiguous');
expect(bookSync).toContain('stats_row_ambiguous = current_stats_row ~= nil and metadata.stats_ambiguous == true');
expect(bookSync).toContain('and not ctx.snap.stats_row_ambiguous');
expect(bookSync).toContain('local body, err = ctx.client:matchCheck({ ctx.snap.digest }, {'); expect(bookSync).toContain('local body, err = ctx.client:matchCheck({ ctx.snap.digest }, {');
expect(bookSync).toContain('title = ctx.snap.title'); expect(bookSync).toContain('title = ctx.snap.title');
expect(bookSync).toContain('authors = ctx.snap.authors'); expect(bookSync).toContain('authors = ctx.snap.authors');
expect(bookSync).toContain('last_open = ctx.snap.last_open'); expect(bookSync).toContain('last_open = ctx.snap.last_open');
expect(bookSync).toContain('source = "current_file"'); expect(bookSync).toContain('source = "current_file"');
expect(bookSync).toContain('book_file_id = book and book.fileId or nil'); expect(bookSync).toContain('book_file_id = matchedFileCandidate(ctx, book)');
expect(bookSync).toContain('if not ctx.snap.stats_row_ambiguous then return book and book.fileId or nil end');
expect(bookSync).toContain('metadata_ambiguous = ctx.snap.metadata_ambiguous'); expect(bookSync).toContain('metadata_ambiguous = ctx.snap.metadata_ambiguous');
expect(bookSync).toContain('if ctx.snap.stats_metadata_ambiguous then'); expect(bookSync).toContain('if ctx.snap.stats_metadata_ambiguous then');
expect(bookSync).toContain('ctx.state:setMatched(match.hash, match.bookFileId, match.bookId, ctx.snap.file)'); expect(bookSync).toContain('ctx.state:setMatched(match.hash, match.bookFileId, match.bookId, ctx.snap.file)');
expect(bookSync).toContain('if ctx.snap.stats_row_ambiguous then upload.bookFileId = stats_target.bookFileId end');
expect(bookSync).not.toContain( expect(bookSync).not.toContain(
'if book then\n if ctx.snap.file and not book.file then\n book.file = ctx.snap.file\n end\n return step(ctx, stepStats)\n end', 'if book then\n if ctx.snap.file and not book.file then\n book.file = ctx.snap.file\n end\n return step(ctx, stepStats)\n end',
); );
@@ -360,12 +364,15 @@ describe('KOReader plugin update source wiring', () => {
expect(api).toContain('source = validMatchSource(cand.source)'); expect(api).toContain('source = validMatchSource(cand.source)');
expect(api).toContain('metadataAmbiguous = optionalBoolean(cand.metadata_ambiguous)'); expect(api).toContain('metadataAmbiguous = optionalBoolean(cand.metadata_ambiguous)');
expect(statsReader).toContain('entry.metadata_ambiguous = (entry._variant_count or 0) > 1'); expect(statsReader).toContain('entry.metadata_ambiguous = (entry._variant_count or 0) > 1');
expect(statsReader).toContain('entry.stats_ambiguous = #entry.ids > 1');
expect(sweep).toContain('source = "statistics"'); expect(sweep).toContain('source = "statistics"');
expect(sweep).toContain('metadata_ambiguous = entry.metadata_ambiguous'); expect(sweep).toContain('metadata_ambiguous = entry.metadata_ambiguous');
expect(sweep).toContain('stats_metadata_ambiguous = entry.metadata_ambiguous'); expect(sweep).toContain('stats_metadata_ambiguous = entry.stats_ambiguous');
expect(sweep).toContain('local file_exists = lfs.attributes(file, "mode") == "file"'); expect(sweep).toContain('local file_exists = lfs.attributes(file, "mode") == "file"');
expect(sweep).toContain('cand.source = "file"'); expect(sweep).toContain('cand.source = "file"');
expect(sweep).toContain('cand.metadata_ambiguous = false'); expect(sweep).toContain('cand.metadata_ambiguous = false');
expect(sweep).toContain('if cand.stat_ids and not cand.stats_metadata_ambiguous then'); expect(sweep).toContain('for _, row in ipairs(cand.stats_rows or {}) do');
expect(sweep).toContain('if item.book_file_id then upload.bookFileId = item.book_file_id end');
expect(sweep).toContain('%1 books need to be opened and synced once to recover their reading history.');
}); });
}); });
@@ -1,4 +1,4 @@
import { BadRequestException, Logger } from '@nestjs/common'; import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { RequestUser } from '../../common/types/request-user'; import type { RequestUser } from '../../common/types/request-user';
@@ -10,6 +10,7 @@ import {
import type { PageStatsUploadDto } from './dto'; import type { PageStatsUploadDto } from './dto';
import type { KoreaderPluginRepository } from './koreader-plugin.repository'; import type { KoreaderPluginRepository } from './koreader-plugin.repository';
import type { KoreaderRepository } from './koreader.repository'; import type { KoreaderRepository } from './koreader.repository';
import type { BookService } from '../book/book.service';
import { KoreaderStatsService } from './koreader-stats.service'; import { KoreaderStatsService } from './koreader-stats.service';
import type { DerivedKoreaderSession } from './koreader-stats.util'; import type { DerivedKoreaderSession } from './koreader-stats.util';
@@ -44,6 +45,7 @@ describe('KoreaderStatsService', () => {
}; };
let pluginRepo: { ingestAndDeriveForBook: ReturnType<typeof vi.fn> }; let pluginRepo: { ingestAndDeriveForBook: ReturnType<typeof vi.fn> };
let achievementEvents: { emit: ReturnType<typeof vi.fn> }; let achievementEvents: { emit: ReturnType<typeof vi.fn> };
let bookService: { verifyFileAccess: ReturnType<typeof vi.fn> };
let service: KoreaderStatsService; let service: KoreaderStatsService;
beforeEach(() => { beforeEach(() => {
@@ -62,11 +64,13 @@ describe('KoreaderStatsService', () => {
.mockResolvedValue({ accepted: 2, duplicates: 0, insertedSessions: [], updatedSessions: [], deletedSessions: 0 }), .mockResolvedValue({ accepted: 2, duplicates: 0, insertedSessions: [], updatedSessions: [], deletedSessions: 0 }),
}; };
achievementEvents = { emit: vi.fn() }; achievementEvents = { emit: vi.fn() };
bookService = { verifyFileAccess: vi.fn() };
service = new KoreaderStatsService( service = new KoreaderStatsService(
koreaderRepo as unknown as KoreaderRepository, koreaderRepo as unknown as KoreaderRepository,
pluginRepo as unknown as KoreaderPluginRepository, pluginRepo as unknown as KoreaderPluginRepository,
achievementEvents as unknown as AchievementEventsService, achievementEvents as unknown as AchievementEventsService,
bookService as unknown as BookService,
); );
}); });
@@ -121,6 +125,108 @@ describe('KoreaderStatsService', () => {
expect(result.results[0]).toEqual({ hash: HASH_A, accepted: 0, duplicates: 2, watermark: 9000 }); expect(result.results[0]).toEqual({ hash: HASH_A, accepted: 0, duplicates: 2, watermark: 9000 });
}); });
it('routes a colliding statistics row to its explicit accessible file when the hash matches', async () => {
koreaderRepo.resolveBookFilesByHashes.mockResolvedValue(new Map());
bookService.verifyFileAccess.mockResolvedValue({
id: 11,
bookId: 21,
libraryId: 1,
format: 'epub',
role: 'content',
fileHash: HASH_A,
});
const dto = makeDto([
{
hash: HASH_A,
bookFileId: 11,
events: [{ page: 1, startTime: 1000, durationSeconds: 30, totalPages: 100 }],
},
]);
const result = await service.uploadPageStats(makeUser(), dto);
expect(bookService.verifyFileAccess).toHaveBeenCalledWith(11, makeUser());
expect(pluginRepo.ingestAndDeriveForBook).toHaveBeenCalledWith(expect.objectContaining({ bookFileId: 11, bookId: 21, libraryId: 1 }));
expect(result.unmatched).toEqual([]);
});
it('refuses an explicit statistics target whose server hash does not match', async () => {
bookService.verifyFileAccess.mockResolvedValue({
id: 11,
bookId: 21,
libraryId: 1,
format: 'epub',
role: 'content',
fileHash: HASH_B,
});
const dto = makeDto([
{
hash: HASH_A,
bookFileId: 11,
events: [{ page: 1, startTime: 1000, durationSeconds: 30, totalPages: 100 }],
},
]);
const result = await service.uploadPageStats(makeUser(), dto);
expect(pluginRepo.ingestAndDeriveForBook).not.toHaveBeenCalled();
expect(result.unmatched).toEqual([HASH_A]);
});
it('refuses an inaccessible explicit statistics target instead of falling back to the hash match', async () => {
bookService.verifyFileAccess.mockRejectedValue(new NotFoundException());
const dto = makeDto([
{
hash: HASH_A,
bookFileId: 11,
events: [{ page: 1, startTime: 1000, durationSeconds: 30, totalPages: 100 }],
},
]);
const result = await service.uploadPageStats(makeUser(), dto);
expect(pluginRepo.ingestAndDeriveForBook).not.toHaveBeenCalled();
expect(result.unmatched).toEqual([HASH_A]);
});
it('refuses a non-content file as an explicit statistics target', async () => {
bookService.verifyFileAccess.mockResolvedValue({
id: 11,
bookId: 21,
libraryId: 1,
format: 'jpg',
role: 'cover',
fileHash: HASH_A,
});
const dto = makeDto([
{
hash: HASH_A,
bookFileId: 11,
events: [{ page: 1, startTime: 1000, durationSeconds: 30, totalPages: 100 }],
},
]);
const result = await service.uploadPageStats(makeUser(), dto);
expect(pluginRepo.ingestAndDeriveForBook).not.toHaveBeenCalled();
expect(result.unmatched).toEqual([HASH_A]);
});
it('does not disguise an unexpected explicit-target lookup failure as an unmatched book', async () => {
const failure = new Error('database unavailable');
bookService.verifyFileAccess.mockRejectedValue(failure);
const dto = makeDto([
{
hash: HASH_A,
bookFileId: 11,
events: [{ page: 1, startTime: 1000, durationSeconds: 30, totalPages: 100 }],
},
]);
await expect(service.uploadPageStats(makeUser(), dto)).rejects.toBe(failure);
expect(pluginRepo.ingestAndDeriveForBook).not.toHaveBeenCalled();
});
it('emits one reading-session event per inserted session below the backfill threshold', async () => { it('emits one reading-session event per inserted session below the backfill threshold', async () => {
pluginRepo.ingestAndDeriveForBook.mockResolvedValue({ pluginRepo.ingestAndDeriveForBook.mockResolvedValue({
accepted: 2, accepted: 2,
@@ -1,4 +1,4 @@
import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import { BadRequestException, HttpException, Injectable, Logger } from '@nestjs/common';
import type { UserSettings } from '@bookorbit/types'; import type { UserSettings } from '@bookorbit/types';
import type { RequestUser } from '../../common/types/request-user'; import type { RequestUser } from '../../common/types/request-user';
@@ -9,6 +9,7 @@ import {
ACHIEVEMENT_EVENT_READING_SESSION_SAVED, ACHIEVEMENT_EVENT_READING_SESSION_SAVED,
AchievementEventsService, AchievementEventsService,
} from '../achievement/achievement-events.service'; } from '../achievement/achievement-events.service';
import { BookService } from '../book/book.service';
import type { PageStatsUploadDto } from './dto'; import type { PageStatsUploadDto } from './dto';
import { KoreaderPluginRepository } from './koreader-plugin.repository'; import { KoreaderPluginRepository } from './koreader-plugin.repository';
import { KoreaderRepository } from './koreader.repository'; import { KoreaderRepository } from './koreader.repository';
@@ -37,6 +38,7 @@ export class KoreaderStatsService {
private readonly koreaderRepo: KoreaderRepository, private readonly koreaderRepo: KoreaderRepository,
private readonly pluginRepo: KoreaderPluginRepository, private readonly pluginRepo: KoreaderPluginRepository,
private readonly achievementEvents: AchievementEventsService, private readonly achievementEvents: AchievementEventsService,
private readonly bookService: BookService,
) {} ) {}
async uploadPageStats(user: RequestUser, dto: PageStatsUploadDto): Promise<PageStatsUploadResult> { async uploadPageStats(user: RequestUser, dto: PageStatsUploadDto): Promise<PageStatsUploadResult> {
@@ -65,7 +67,23 @@ export class KoreaderStatsService {
for (const book of dto.books) { for (const book of dto.books) {
const hash = book.hash.toLowerCase(); const hash = book.hash.toLowerCase();
const match = matches.get(hash); let match = matches.get(hash);
if (book.bookFileId) {
const explicitFile = await this.bookService.verifyFileAccess(book.bookFileId, user).catch((error: unknown) => {
if (error instanceof HttpException) return null;
throw error;
});
if (explicitFile?.role === 'content' && explicitFile.fileHash?.toLowerCase() === hash) {
match = {
bookFileId: explicitFile.id,
bookId: explicitFile.bookId,
libraryId: explicitFile.libraryId,
format: explicitFile.format,
};
} else {
match = undefined;
}
}
if (!match) { if (!match) {
unmatched.push(hash); unmatched.push(hash);
continue; continue;
+15 -3
View File
@@ -53,7 +53,7 @@ describe('KOReader page stats derivation (e2e)', { timeout: 180_000 }, () => {
return { 'x-auth-user': KOREADER_USERNAME, 'x-auth-key': KOREADER_PASSWORD }; return { 'x-auth-user': KOREADER_USERNAME, 'x-auth-key': KOREADER_PASSWORD };
} }
async function uploadPageStats(events: KoreaderPageEvent[]) { async function uploadPageStats(events: KoreaderPageEvent[], bookFileId?: number) {
const response = await ctx.app.inject({ const response = await ctx.app.inject({
method: 'POST', method: 'POST',
url: '/api/v1/koreader/plugin/page-stats', url: '/api/v1/koreader/plugin/page-stats',
@@ -62,7 +62,7 @@ describe('KOReader page stats derivation (e2e)', { timeout: 180_000 }, () => {
deviceId: DEVICE_ID, deviceId: DEVICE_ID,
deviceModel: 'E2E', deviceModel: 'E2E',
pluginVersion: '0.4.0', pluginVersion: '0.4.0',
books: [{ hash: fileHash, events }], books: [{ hash: fileHash, bookFileId, events }],
}, },
}); });
expect(response.statusCode).toBe(201); expect(response.statusCode).toBe(201);
@@ -310,10 +310,22 @@ describe('KOReader page stats derivation (e2e)', { timeout: 180_000 }, () => {
await uploadPageStats([{ page: 1, startTime: firstStart, durationSeconds: 55, totalPages: TOTAL_PAGES }]); await uploadPageStats([{ page: 1, startTime: firstStart, durationSeconds: 55, totalPages: TOTAL_PAGES }]);
await uploadPageStats([{ page: 2, startTime: firstStart + 60, durationSeconds: 55, totalPages: TOTAL_PAGES }]); await uploadPageStats([{ page: 2, startTime: firstStart + 60, durationSeconds: 55, totalPages: TOTAL_PAGES }]);
const explicitStart = firstStart + 120;
const explicit = await uploadPageStats([{ page: 3, startTime: explicitStart, durationSeconds: 55, totalPages: TOTAL_PAGES }], sharing[1]!.id);
expect(explicit.results).toEqual([{ hash: fileHash, accepted: 1, duplicates: 0, watermark: explicitStart }]);
const [explicitRow] = await ctx.db
.select({ bookFileId: schema.koreaderPageStats.bookFileId })
.from(schema.koreaderPageStats)
.where(and(eq(schema.koreaderPageStats.deviceId, DEVICE_ID), eq(schema.koreaderPageStats.startTime, explicitStart)));
expect(explicitRow?.bookFileId).toBe(sharing[1]!.id);
const targets = await ctx.db const targets = await ctx.db
.selectDistinct({ bookFileId: schema.koreaderPageStats.bookFileId }) .selectDistinct({ bookFileId: schema.koreaderPageStats.bookFileId })
.from(schema.koreaderPageStats) .from(schema.koreaderPageStats)
.where(and(eq(schema.koreaderPageStats.deviceId, DEVICE_ID), gte(schema.koreaderPageStats.startTime, firstStart))); .where(and(eq(schema.koreaderPageStats.deviceId, DEVICE_ID), gte(schema.koreaderPageStats.startTime, firstStart)));
expect(targets.map((row) => row.bookFileId)).toEqual([epub.bookFileId]); expect(targets.map((row) => row.bookFileId).sort((left, right) => left - right)).toEqual(
[epub.bookFileId, sharing[1]!.id].sort((left, right) => left - right),
);
}); });
}); });