fix(series): load complete KOReader series lists

Fetch every bounded API page before applying local pagination so large series remain complete in KOReader and available offline.

Closes #1883

koreader-plugin/bookorbit.koplugin/bookorbit_catalog.lua

@@ -1124,7 +1124,6 @@ function BookOrbitCatalog:paramsForEntry(section, entry)
         else
             params.series = entry.id
         end
-        params.sort = "series"
     end
     return params
 end
@@ -1143,23 +1142,40 @@ function BookOrbitCatalog:fetchBookPage(params, title, push)
     local body, err = self:fetch(title and T(_("Opening %1…"), title) or _("Opening books…"), function()
         return self.client:catalogBooks(query)
     end)
+    local offline_saved_at
     if not body then
         if err ~= "cancelled" then
             -- Unreachable: the page this device saved for the same request.
             local saved = self:listSavedPage(query, err)
             if saved then
-                self:showBookPage(saved.body, query, title or _("Books"), push ~= false, { offline_saved_at = saved.saved_at or 0 })
+                body = saved.body
-                return
+                offline_saved_at = saved.saved_at or 0
+            else
+                self:showRetry(err, function()
+                    self:loadBooks(params, title, push)
+                end)
             end
-            self:showRetry(err, function()
+        end
+        if not body then return end
+    else
+        self:listSavePage(query, body)
+    end
+
+    local complete, complete_err, tail_saved_at = self:listFetchCompleteSeries(query, body, title)
+    if not complete then
+        if complete_err ~= "cancelled" then
+            self:showRetry(complete_err, function()
                 self:loadBooks(params, title, push)
             end)
         end
         return
     end
-
+    if tail_saved_at then
-    self:listSavePage(query, body)
+        offline_saved_at = offline_saved_at and math.min(offline_saved_at, tail_saved_at) or tail_saved_at
-    self:showBookPage(body, query, title or _("Books"), push ~= false)
+    end
+    local complete_series = query.seriesId ~= nil or query.series ~= nil
+    self:showBookPage(complete, query, title or _("Books"), push ~= false,
+        offline_saved_at and { offline_saved_at = offline_saved_at } or { complete = complete_series })
 end
 
 function BookOrbitCatalog:loadBooks(params, title, push)
@@ -1387,7 +1403,7 @@ function BookOrbitCatalog:showBookPage(body, query, title, push, opts)
         list_saved_at = opts.offline_saved_at,
     }, push)
 
-    if not opts.offline_saved_at then
+    if not opts.offline_saved_at and not opts.complete then
         if offset then
             if offset + size < total then self:prefetchNextPage(stored, nil, offset + size) end
         elseif page < page_count then

koreader-plugin/bookorbit.koplugin/bookorbit_catalog_detail.lua

@@ -54,11 +54,11 @@ end
 function CatalogDetail.detailSeriesParams(detail)
     if not detail then return nil end
     if detail.seriesId then
-        return { seriesId = tonumber(detail.seriesId), sort = "series" }
+        return { seriesId = tonumber(detail.seriesId), sort = "title" }
     end
     local name = cleanInlineText(detail.seriesName)
     if not name then return nil end
-    return { series = name, sort = "series" }
+    return { series = name, sort = "title" }
 end
 
 function CatalogDetail.detailSeriesText(detail)

koreader-plugin/bookorbit.koplugin/bookorbit_catalog_lists.lua

@@ -152,6 +152,69 @@ function CatalogLists:listSavedPage(query, err)
     return ok and saved or nil
 end
 
+-- A series is drawn as one local list, while the API deliberately caps each
+-- response. Keep requesting bounded server pages before the local paginator
+-- takes over.
+function CatalogLists:listFetchCompleteSeries(query, first_body, title)
+    if not (query.seriesId or query.series) then return first_body end
+
+    local items, seen = {}, {}
+    local function append(body)
+        local added = 0
+        for _i, book in ipairs(body.items or {}) do
+            local key = book.id ~= nil and tostring(book.id) or nil
+            if not key or not seen[key] then
+                if key then seen[key] = true end
+                table.insert(items, book)
+                added = added + 1
+            end
+        end
+        return added
+    end
+
+    append(first_body)
+    local body = first_body
+    local total = tonumber(body.total)
+    local page = tonumber(body.page) or tonumber(query.page) or 1
+    local offline_saved_at
+    local has_next = body.hasNext == true or (total and #items < total)
+
+    while has_next do
+        page = page + 1
+        local page_query = cloneParams(query)
+        page_query.page = page
+        page_query.offset = nil
+        local next_body, err = self:fetch(title and T(_("Opening %1…"), title) or _("Opening books…"), function()
+            return self.client:catalogBooks(page_query)
+        end)
+        if not next_body then
+            local saved = self:listSavedPage(page_query, err)
+            if not saved then return nil, err end
+            next_body = saved.body
+            local saved_at = tonumber(saved.saved_at) or 0
+            offline_saved_at = offline_saved_at and math.min(offline_saved_at, saved_at) or saved_at
+        else
+            self:listSavePage(page_query, next_body)
+        end
+
+        if append(next_body) == 0 then return nil, "incomplete series response" end
+        body = next_body
+        total = tonumber(body.total) or total
+        has_next = body.hasNext == true or (total and #items < total)
+    end
+
+    local complete = {}
+    for key, value in pairs(first_body) do complete[key] = value end
+    complete.items = items
+    complete.total = math.max(total or 0, #items)
+    complete.page = 1
+    complete.hasNext = false
+    complete.hasPrevious = false
+    complete.nextUrl = nil
+    complete.previousUrl = nil
+    return complete, nil, offline_saved_at
+end
+
 -- The server page size for a book list. A series or an author page draws
 -- all its books (local pages when there are more than fit).
 function CatalogLists:listBookPageSize(params)
@@ -1162,6 +1225,7 @@ function CatalogLists:listBody(kind, context)
         local m = {
             entries = books, view = self:listView(kind), screen_w = sw, screen_h = sh, local_page = context.local_page,
             offline = context.list_offline and {} or nil,
+            order_label = (context.params or {}).sort == "title" and _("Title A–Z") or _("Reading order"),
             states = states, indexes = indexes, current_index = cur, next_book = next_book,
             range = #books > 0 and T(_("Books %1 – %2"), first, last) or nil,
             next_minutes = next_book and self:listMinutesLeft(next_book),

koreader-plugin/bookorbit.koplugin/bookorbit_list_pages.lua

@@ -231,7 +231,7 @@ function Pages.seriesBody(m, f)
     else
         hero = hstack({ w = f.w, h = hero_h, gap = px(20) }, { deck, vstack({ w = tw, h = hero_h }, { flex(0), summary_col, flex(0) }) })
     end
-    local head = ListView.sectionHead(list_w, { icon = "layers-2", label = _("Reading order"), count = #all_books, right = ListView.viewToggle(m.view, "series-head") })
+    local head = ListView.sectionHead(list_w, { icon = "layers-2", label = m.order_label or _("Reading order"), count = #all_books, right = ListView.viewToggle(m.view, "series-head") })
     local list_h = page == 1 and layout.list_h1 or layout.list_hn
     local content
     if m.view == "grid" then

koreader-plugin/spec/bookorbit_detail_info_rows_test.lua

@@ -104,7 +104,7 @@ assertEqual(CatalogDetail.detailSeriesText({ seriesIndex = 3 }), nil, "an index
 
 local by_id = CatalogDetail.detailSeriesParams({ seriesId = 12, seriesName = "Discworld" })
 assertEqual(by_id.seriesId, 12, "a series id drives the books context")
-assertEqual(by_id.sort, "series", "a series list sorts by series order")
+assertEqual(by_id.sort, "title", "a series list sorts by title")
 local by_name = CatalogDetail.detailSeriesParams({ seriesName = "Discworld" })
 assertEqual(by_name.series, "Discworld", "without an id the name drives the context")
 assertEqual(by_name.seriesId, nil, "no id is invented")

koreader-plugin/spec/bookorbit_series_paging_test.lua

@@ -0,0 +1,44 @@
+package.path = "koreader-plugin/bookorbit.koplugin/?.lua;" .. package.path
+package.loaded["gettext"] = function(text) return text end
+package.loaded["ffi/util"] = { template = function(text, value) return (text:gsub("%%1", value or "")) end }
+package.loaded["document/documentregistry"] = {}
+package.loaded["util"] = {}
+for _, name in ipairs({ "device", "ui/widget/infomessage", "ui/network/manager", "ui/uimanager",
+    "datetime", "logger", "bookorbit_board_data", "bookorbit_board_metrics",
+    "bookorbit_capabilities", "bookorbit_list_cache", "bookorbit_list_data",
+    "bookorbit_list_pages", "bookorbit_list_view" }) do package.loaded[name] = {} end
+
+local Lists = require("bookorbit_catalog_lists")
+
+local requested, saved = {}, {}
+local catalog = setmetatable({
+    client = {
+        catalogBooks = function(_, query)
+            table.insert(requested, query.page)
+            local first = (query.page - 1) * 100 + 1
+            local last = math.min(135, first + 99)
+            local items = {}
+            for id = first, last do table.insert(items, { id = id, title = string.format("Volume %03d", id) }) end
+            return { items = items, total = 135, page = query.page, size = 100, hasNext = last < 135 }
+        end,
+    },
+}, { __index = Lists })
+function catalog:fetch(_label, fn) return fn() end
+function catalog:listSavePage(query) table.insert(saved, query.page) end
+function catalog:listSavedPage() return nil end
+
+local first = catalog.client:catalogBooks({ page = 1 })
+local complete = assert(catalog:listFetchCompleteSeries({ page = 1, size = 100, seriesId = 42, sort = "title" }, first, "Series"))
+assert(#complete.items == 135, "a series joins every API page")
+assert(complete.items[101].title == "Volume 101", "the next page keeps title order")
+assert(complete.hasNext == false and complete.nextUrl == nil, "the joined response is complete")
+assert(#requested == 2 and requested[2] == 2, "only the missing page is requested")
+assert(#saved == 1 and saved[1] == 2, "the added page is cached independently")
+
+requested = {}
+local ordinary = { items = { { id = 1 } }, total = 200, page = 1, size = 100, hasNext = true }
+assert(catalog:listFetchCompleteSeries({ page = 1, size = 100, sort = "title" }, ordinary) == ordinary,
+    "ordinary book lists keep server pagination")
+assert(#requested == 0, "ordinary book lists do not fetch ahead")
+
+print("bookorbit_series_paging_test.lua: ok")