From a24dc2e2cc9bda721339cb347cf3910dbec2c1dd Mon Sep 17 00:00:00 2001 From: neon <18559658+neonsolstice@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:39:10 +0000 Subject: [PATCH] feat(annotations): add a library-wide notebook for every book Add read-only /notebook routes that list highlights, journal entries, bookmarks and reviews across every book a reader can open, as one cursor-paged stream with per-kind counts, a By Book listing, a daily review deck, a seeded review shuffle, On this day and a library-wide trash. Each kind is read through its own index-ordered arm cut at the page size before a UNION ALL merge, so a deep page costs what the first does, and book labels travel once per page in a sidecar map. Scope follows library access and content filters. Everything is additive and advertised as the notebook-hub app-info feature. The iOS app calls these routes and the web client does not yet, so they look unused from the client alone. No migration: the extra indexes for the edited sort and search wait until the chapter-editor migrations land. --- packages/types/src/app-info.ts | 2 +- packages/types/src/index.ts | 1 + packages/types/src/notebook.ts | 194 +++++++ scripts/e2e/suite-registry.mjs | 26 + server/src/app.module.ts | 2 + .../dto/notebook-dto-validation.test.ts | 127 +++++ .../notebook/dto/notebook-query.dto.ts | 212 ++++++++ .../modules/notebook/notebook-cursor.test.ts | 47 ++ .../src/modules/notebook/notebook-cursor.ts | 47 ++ .../modules/notebook/notebook-dates.test.ts | 72 +++ server/src/modules/notebook/notebook-dates.ts | 71 +++ .../modules/notebook/notebook-filters.test.ts | 74 +++ .../src/modules/notebook/notebook-filters.ts | 91 ++++ .../modules/notebook/notebook-mapper.test.ts | 190 +++++++ .../src/modules/notebook/notebook-mapper.ts | 163 ++++++ .../src/modules/notebook/notebook-sql.test.ts | 236 ++++++++ server/src/modules/notebook/notebook-sql.ts | 444 +++++++++++++++ .../notebook/notebook.controller.test.ts | 74 +++ .../modules/notebook/notebook.controller.ts | 58 ++ .../modules/notebook/notebook.module.test.ts | 18 + .../src/modules/notebook/notebook.module.ts | 14 + .../modules/notebook/notebook.repository.ts | 354 ++++++++++++ .../modules/notebook/notebook.service.test.ts | 387 +++++++++++++ .../src/modules/notebook/notebook.service.ts | 296 ++++++++++ server/test/notebook-hub.e2e-spec.ts | 507 ++++++++++++++++++ 25 files changed, 3706 insertions(+), 1 deletion(-) create mode 100644 packages/types/src/notebook.ts create mode 100644 server/src/modules/notebook/dto/notebook-dto-validation.test.ts create mode 100644 server/src/modules/notebook/dto/notebook-query.dto.ts create mode 100644 server/src/modules/notebook/notebook-cursor.test.ts create mode 100644 server/src/modules/notebook/notebook-cursor.ts create mode 100644 server/src/modules/notebook/notebook-dates.test.ts create mode 100644 server/src/modules/notebook/notebook-dates.ts create mode 100644 server/src/modules/notebook/notebook-filters.test.ts create mode 100644 server/src/modules/notebook/notebook-filters.ts create mode 100644 server/src/modules/notebook/notebook-mapper.test.ts create mode 100644 server/src/modules/notebook/notebook-mapper.ts create mode 100644 server/src/modules/notebook/notebook-sql.test.ts create mode 100644 server/src/modules/notebook/notebook-sql.ts create mode 100644 server/src/modules/notebook/notebook.controller.test.ts create mode 100644 server/src/modules/notebook/notebook.controller.ts create mode 100644 server/src/modules/notebook/notebook.module.test.ts create mode 100644 server/src/modules/notebook/notebook.module.ts create mode 100644 server/src/modules/notebook/notebook.repository.ts create mode 100644 server/src/modules/notebook/notebook.service.test.ts create mode 100644 server/src/modules/notebook/notebook.service.ts create mode 100644 server/test/notebook-hub.e2e-spec.ts diff --git a/packages/types/src/app-info.ts b/packages/types/src/app-info.ts index f308d435f..e2ef5c268 100644 --- a/packages/types/src/app-info.ts +++ b/packages/types/src/app-info.ts @@ -5,7 +5,7 @@ * * Named apart from `APP_FEATURES` in app-features.ts, which holds the build-time feature flags. */ -export const SERVER_FEATURES = ["annotation-stars", "annotation-color-names", "annotation-trash", "book-journal", "bookmark-edit"] as const; +export const SERVER_FEATURES = ["annotation-stars", "annotation-color-names", "annotation-trash", "book-journal", "bookmark-edit", "notebook-hub"] as const; export type ServerFeature = (typeof SERVER_FEATURES)[number]; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 99c9da05d..fa10e0f16 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -73,6 +73,7 @@ export * from "./reading-session-source-bucket"; export * from "./shared-reading-insights"; export * from "./annotation"; export * from "./book-journal"; +export * from "./notebook"; export * from "./hardcover"; export * from "./readwise"; export * from "./storygraph"; diff --git a/packages/types/src/notebook.ts b/packages/types/src/notebook.ts new file mode 100644 index 000000000..2b5e0d276 --- /dev/null +++ b/packages/types/src/notebook.ts @@ -0,0 +1,194 @@ +import type { AnnotationItem, AnnotationPositionStatus } from "./annotation"; + +/** + * The library-wide Notebook: every highlight, journal entry, bookmark and review a reader has + * written, across every book they can still open, as one cursor-paged stream. Served under + * `/notebook` and advertised as the `notebook-hub` server feature. + * + * Nothing here is ever loaded whole. A page carries at most `NOTEBOOK_PAGE_SIZE_MAX` entries, the + * counts come from their own route, and the cursor keeps page 200 as cheap as page 1. + */ + +export const NOTEBOOK_ENTRY_KINDS = ["highlight", "journal", "bookmark", "review"] as const; +export type NotebookEntryKind = (typeof NOTEBOOK_ENTRY_KINDS)[number]; + +/** `newest` and `oldest` order by when an entry was written, `edited` by when it last changed. */ +export const NOTEBOOK_SORTS = ["newest", "oldest", "edited"] as const; +export type NotebookSort = (typeof NOTEBOOK_SORTS)[number]; + +export const NOTEBOOK_PAGE_SIZE_DEFAULT = 40; +export const NOTEBOOK_PAGE_SIZE_MAX = 100; +export const NOTEBOOK_SEARCH_MAX_LENGTH = 200; +export const NOTEBOOK_BOOKS_PAGE_SIZE_DEFAULT = 30; +export const NOTEBOOK_BOOKS_PAGE_SIZE_MAX = 60; +export const NOTEBOOK_REVIEW_DAILY_SIZE = 10; +export const NOTEBOOK_REVIEW_SHUFFLE_MAX = 200; +export const NOTEBOOK_ON_THIS_DAY_MAX = 20; +/** Colours a book row reports, heaviest first. */ +export const NOTEBOOK_BOOK_COLOR_MIX_MAX = 5; + +/** Who a book is. Sent once per page in a sidecar map, never repeated on every entry. */ +export interface NotebookBookLabel { + id: number; + title: string | null; + /** Primary authors in display order, joined with ", ". */ + author: string | null; + hasCover: boolean; + /** The same cache key the library's book cards carry, so a cover cached there is reused here. */ + coverVersion: string | null; + /** `"1/1"` for square (audiobook) art, else the book's ratio such as `"2/3"`. */ + coverAspectRatio: string | null; + /** The file Open in Book opens (the book's primary readable file), or null when there is none. */ + readFileId: number | null; + /** That file's format in lower case (`epub`, `pdf`, `cbz`, ...), or null. */ + readFileFormat: string | null; + hasAudio: boolean; +} + +interface NotebookEntryBase { + kind: NotebookEntryKind; + /** Server id within its kind. A review has no row id of its own and uses its book id. */ + id: number; + bookId: number; + /** The client UUID a journal entry or bookmark is addressed by; null for highlights and reviews. */ + clientId: string | null; + /** When it was written. A highlight uses when it was highlighted, which a device may backdate. */ + createdAt: string; + updatedAt: string; + /** Set only on items from `GET /notebook/trash`. */ + deletedAt?: string | null; +} + +export interface NotebookHighlightEntry extends NotebookEntryBase { + kind: "highlight"; + text: string; + color: string; + style: string; + note: string | null; + chapterTitle: string | null; + cfi: string | null; + positionStatus: AnnotationPositionStatus | null; + origin: AnnotationItem["origin"]; + starredAt: string | null; +} + +export interface NotebookJournalItem extends NotebookEntryBase { + kind: "journal"; + clientId: string; + body: string; + quote: string | null; + chapterTitle: string | null; + /** 0 to 100. */ + positionPercent: number | null; + cfi: string | null; + positionSeconds: number | null; +} + +/** An EPUB bookmark carries a CFI; an audiobook bookmark carries a time instead. */ +export interface NotebookBookmarkItem extends NotebookEntryBase { + kind: "bookmark"; + clientId: string; + title: string; + note: string | null; + cfi: string | null; + positionSeconds: number | null; + origin: string; +} + +/** A book's My review. `createdAt` equals `updatedAt`: the server keeps only the latest edit time. */ +export interface NotebookReviewItem extends NotebookEntryBase { + kind: "review"; + body: string; + /** The reader's own star rating for the book, 1 to 5, or null when unrated. */ + rating: number | null; +} + +export type NotebookEntry = NotebookHighlightEntry | NotebookJournalItem | NotebookBookmarkItem | NotebookReviewItem; + +/** + * `GET /notebook/entries`. + * + * Filters: `kinds` (comma list, default all), `starred`, `colors` (comma list of stored colour + * strings), `hasNote`, `approximate`, `origins` (comma list), `bookId`, `q`, `sort`, `cursor`, + * `limit`. Highlight-only filters (`starred`, `colors`, `approximate`) leave out every other kind + * while set. `origins` applies to highlights and bookmarks; journal entries and reviews count as + * `web`. `hasNote` keeps highlights and bookmarks with a note plus every journal entry and review. + * `q` matches the entry's own text and its book's title and authors, accent-insensitively. + */ +export interface NotebookEntriesResponse { + items: NotebookEntry[]; + /** Keyed by book id as a string. */ + books: Record; + /** Pass back as `cursor` for the next page; null on the last page. */ + nextCursor: string | null; +} + +/** `GET /notebook/overview`: the counts behind the chips and rail, for the same filters minus `kinds`. */ +export interface NotebookOverview { + total: number; + highlights: number; + journal: number; + bookmarks: number; + reviews: number; + starred: number; + withNotes: number; + approximate: number; + /** Books with at least one counted entry. */ + books: number; + colors: { color: string; count: number }[]; + origins: { origin: AnnotationItem["origin"]; count: number }[]; +} + +export interface NotebookBookRow { + book: NotebookBookLabel; + highlights: number; + journal: number; + bookmarks: number; + hasReview: boolean; + /** Up to `NOTEBOOK_BOOK_COLOR_MIX_MAX` highlight colours, heaviest first. */ + colors: { color: string; count: number }[]; + /** The newest entry's write time. */ + lastActivityAt: string; + /** The newest entry, for the row's one-line excerpt. */ + latest: NotebookEntry | null; +} + +/** `GET /notebook/books`: books with entries, most recent activity first. Query: `q`, `cursor`, `limit`. */ +export interface NotebookBooksResponse { + items: NotebookBookRow[]; + nextCursor: string | null; +} + +/** + * `GET /notebook/review`. With `date=YYYY-MM-DD`, the day's deck: `NOTEBOOK_REVIEW_DAILY_SIZE` + * highlights from different books, starred first, then older ones, stable for that day. With + * `seed` plus the entry filters, up to `NOTEBOOK_REVIEW_SHUFFLE_MAX` matching highlights in a + * shuffled order that the same seed repeats. + */ +export interface NotebookReviewResponse { + items: NotebookHighlightEntry[]; + books: Record; +} + +export interface NotebookOnThisDayYear { + year: number; + items: NotebookEntry[]; +} + +/** + * `GET /notebook/on-this-day?date=YYYY-MM-DD&tz=`: entries written on that month and + * day in earlier years, newest year first, at most `NOTEBOOK_ON_THIS_DAY_MAX` in all. `tz` + * defaults to the reader's timezone setting. + */ +export interface NotebookOnThisDayResponse { + date: string; + years: NotebookOnThisDayYear[]; + books: Record; +} + +/** `GET /notebook/trash`: deleted highlights and journal entries across books, newest deletion first. */ +export interface NotebookTrashResponse { + items: (NotebookHighlightEntry | NotebookJournalItem)[]; + books: Record; + nextCursor: string | null; +} diff --git a/scripts/e2e/suite-registry.mjs b/scripts/e2e/suite-registry.mjs index 3f5e199c2..cc6f6f070 100644 --- a/scripts/e2e/suite-registry.mjs +++ b/scripts/e2e/suite-registry.mjs @@ -853,6 +853,32 @@ export const E2E_SUITES = Object.freeze({ ...SHARED_TYPES_PATHS, ], }, + "notebook-hub": { + id: "notebook-hub", + name: "Notebook Hub", + timeout: 45, + lane: "full", + description: "Library-wide notebook: entries, overview, books, review decks, on this day and trash suite", + vitestTarget: "test/notebook-hub.e2e-spec.ts", + junitOutput: `${TEST_RESULTS_DIR}/notebook-hub-e2e-junit.xml`, + prepareDedicatedDatabase: true, + useDedicatedDatabase: true, + changedPaths: [ + "server/src/modules/notebook/**", + "server/src/modules/annotation/**", + "server/src/modules/book-journal/**", + "server/src/modules/bookmark/**", + "server/src/modules/user-book-note/**", + "server/src/modules/book-cover-store/**", + "server/src/modules/library/**", + "server/src/common/utils/content-filter-sql.utils.ts", + "server/src/common/utils/accent-insensitive-search.utils.ts", + "server/test/notebook-hub.e2e-spec.ts", + "server/test/e2e/reader-state-isolation/**", + ...SHARED_DB_AND_HELPER_PATHS, + ...SHARED_TYPES_PATHS, + ], + }, "kobo-annotation-sync": { id: "kobo-annotation-sync", name: "Kobo Annotation Sync", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index ca0ca60d8..c01177f29 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -38,6 +38,7 @@ import { AuthModule } from './modules/auth/auth.module'; import { CbzModule } from './modules/reader/cbz/cbz.module'; import { BookmarkModule } from './modules/bookmark/bookmark.module'; import { BookJournalModule } from './modules/book-journal/book-journal.module'; +import { NotebookModule } from './modules/notebook/notebook.module'; import { BookModule } from './modules/book/book.module'; import { CatalogModule } from './modules/catalog/catalog.module'; import { CoverModule } from './modules/cover/cover.module'; @@ -160,6 +161,7 @@ import { PodcastModule } from './modules/podcast/podcast.module'; BookmarkModule, BookJournalModule, AnnotationModule, + NotebookModule, DashboardModule, HealthModule, CbzModule, diff --git a/server/src/modules/notebook/dto/notebook-dto-validation.test.ts b/server/src/modules/notebook/dto/notebook-dto-validation.test.ts new file mode 100644 index 000000000..a9f30cdf8 --- /dev/null +++ b/server/src/modules/notebook/dto/notebook-dto-validation.test.ts @@ -0,0 +1,127 @@ +import 'reflect-metadata'; + +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; + +import { NOTEBOOK_BOOKS_PAGE_SIZE_MAX, NOTEBOOK_PAGE_SIZE_MAX, NOTEBOOK_SEARCH_MAX_LENGTH } from '@bookorbit/types'; + +import { + NotebookBooksQueryDto, + NotebookEntriesQueryDto, + NotebookOnThisDayQueryDto, + NotebookOverviewQueryDto, + NotebookReviewQueryDto, + NotebookTrashQueryDto, +} from './notebook-query.dto'; + +async function check(cls: new () => T, value: Record) { + const dto = plainToInstance(cls, value); + const errors = await validate(dto, { whitelist: true, forbidNonWhitelisted: true }); + return { dto, errors }; +} + +describe('NotebookEntriesQueryDto', () => { + it('accepts an empty query', async () => { + expect((await check(NotebookEntriesQueryDto, {})).errors).toHaveLength(0); + }); + + it('converts query-string booleans, so false means no filter rather than true', async () => { + const { dto, errors } = await check(NotebookEntriesQueryDto, { starred: 'true', hasNote: 'false', approximate: 'false' }); + expect(errors).toHaveLength(0); + expect(dto.starred).toBe(true); + expect(dto.hasNote).toBe(false); + expect(dto.approximate).toBe(false); + }); + + it('rejects anything else as a boolean', async () => { + expect((await check(NotebookEntriesQueryDto, { starred: '1' })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { hasNote: 'yes' })).errors).toHaveLength(1); + }); + + it('parses comma and repeated lists for kinds and origins', async () => { + const { dto, errors } = await check(NotebookEntriesQueryDto, { kinds: 'highlight, review,highlight', origins: ['kobo', 'web,kobo'] }); + expect(errors).toHaveLength(0); + expect(dto.kinds).toEqual(['highlight', 'review']); + expect(dto.origins).toEqual(['kobo', 'web']); + }); + + it('rejects an unknown kind or origin', async () => { + expect((await check(NotebookEntriesQueryDto, { kinds: 'highlight,podcast' })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { origins: 'web,apple' })).errors).toHaveLength(1); + }); + + it('bounds limit, q, colors, cursor and bookId', async () => { + expect((await check(NotebookEntriesQueryDto, { limit: '0' })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { limit: String(NOTEBOOK_PAGE_SIZE_MAX + 1) })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { limit: 'ten' })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { q: 'x'.repeat(NOTEBOOK_SEARCH_MAX_LENGTH + 1) })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { colors: 'x'.repeat(301) })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { cursor: 'x'.repeat(513) })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { bookId: '0' })).errors).toHaveLength(1); + expect((await check(NotebookEntriesQueryDto, { bookId: '2147483648' })).errors).toHaveLength(1); + + const { dto, errors } = await check(NotebookEntriesQueryDto, { limit: '100', bookId: '12', q: ' dune ' }); + expect(errors).toHaveLength(0); + expect(dto).toMatchObject({ limit: 100, bookId: 12, q: 'dune' }); + }); + + it('accepts only the documented sorts', async () => { + expect((await check(NotebookEntriesQueryDto, { sort: 'edited' })).errors).toHaveLength(0); + expect((await check(NotebookEntriesQueryDto, { sort: 'random' })).errors).toHaveLength(1); + }); + + it('forbids unknown parameters', async () => { + expect((await check(NotebookEntriesQueryDto, { page: '2' })).errors).toHaveLength(1); + }); +}); + +describe('NotebookOverviewQueryDto', () => { + it('takes the filters but not kinds, sort, cursor or limit', async () => { + expect((await check(NotebookOverviewQueryDto, { starred: 'true', colors: '#FACC15', q: 'x', bookId: '3' })).errors).toHaveLength(0); + for (const extra of [{ kinds: 'highlight' }, { sort: 'newest' }, { cursor: 'abc' }, { limit: '5' }]) { + expect((await check(NotebookOverviewQueryDto, extra)).errors).toHaveLength(1); + } + }); +}); + +describe('NotebookBooksQueryDto', () => { + it('bounds limit to the books page size', async () => { + expect((await check(NotebookBooksQueryDto, { limit: String(NOTEBOOK_BOOKS_PAGE_SIZE_MAX) })).errors).toHaveLength(0); + expect((await check(NotebookBooksQueryDto, { limit: String(NOTEBOOK_BOOKS_PAGE_SIZE_MAX + 1) })).errors).toHaveLength(1); + expect((await check(NotebookBooksQueryDto, { starred: 'true' })).errors).toHaveLength(1); + }); +}); + +describe('NotebookReviewQueryDto', () => { + it('accepts a calendar date or an integer seed with filters', async () => { + expect((await check(NotebookReviewQueryDto, { date: '2028-02-29' })).errors).toHaveLength(0); + const { dto, errors } = await check(NotebookReviewQueryDto, { seed: '-17', colors: 'blue', starred: 'true' }); + expect(errors).toHaveLength(0); + expect(dto.seed).toBe(-17); + }); + + it('rejects impossible dates and non-integer seeds', async () => { + expect((await check(NotebookReviewQueryDto, { date: '2026-02-29' })).errors).toHaveLength(1); + expect((await check(NotebookReviewQueryDto, { date: '2026-1-5' })).errors).toHaveLength(1); + expect((await check(NotebookReviewQueryDto, { seed: '1.5' })).errors).toHaveLength(1); + expect((await check(NotebookReviewQueryDto, { seed: 'abc' })).errors).toHaveLength(1); + expect((await check(NotebookReviewQueryDto, { kinds: 'journal', seed: '1' })).errors).toHaveLength(1); + }); +}); + +describe('NotebookOnThisDayQueryDto', () => { + it('requires a date and validates the zone', async () => { + expect((await check(NotebookOnThisDayQueryDto, {})).errors).toHaveLength(1); + expect((await check(NotebookOnThisDayQueryDto, { date: '2026-10-04' })).errors).toHaveLength(0); + expect((await check(NotebookOnThisDayQueryDto, { date: '2026-10-04', tz: 'Asia/Kolkata' })).errors).toHaveLength(0); + expect((await check(NotebookOnThisDayQueryDto, { date: '2026-10-04', tz: 'Mars/Olympus' })).errors).toHaveLength(1); + expect((await check(NotebookOnThisDayQueryDto, { date: '2026-10-04', tz: '' })).errors).toHaveLength(1); + }); +}); + +describe('NotebookTrashQueryDto', () => { + it('takes a cursor and a limit only', async () => { + expect((await check(NotebookTrashQueryDto, { cursor: 'abc', limit: '20' })).errors).toHaveLength(0); + expect((await check(NotebookTrashQueryDto, { kinds: 'highlight' })).errors).toHaveLength(1); + }); +}); diff --git a/server/src/modules/notebook/dto/notebook-query.dto.ts b/server/src/modules/notebook/dto/notebook-query.dto.ts new file mode 100644 index 000000000..ad3e8dc51 --- /dev/null +++ b/server/src/modules/notebook/dto/notebook-query.dto.ts @@ -0,0 +1,212 @@ +import { Transform, Type, type TransformFnParams } from 'class-transformer'; +import { + ArrayMaxSize, + IsArray, + IsBoolean, + IsIn, + IsInt, + IsOptional, + IsString, + Max, + MaxLength, + Min, + ValidateBy, + type ValidationOptions, +} from 'class-validator'; + +import { + NOTEBOOK_BOOKS_PAGE_SIZE_MAX, + NOTEBOOK_ENTRY_KINDS, + NOTEBOOK_PAGE_SIZE_MAX, + NOTEBOOK_SEARCH_MAX_LENGTH, + NOTEBOOK_SORTS, + type NotebookEntryKind, + type NotebookSort, +} from '@bookorbit/types'; + +import { isDateKey, isValidTimeZone } from '../../../common/utils/timezone.utils'; +import { NOTEBOOK_ORIGINS, type NotebookOrigin } from '../notebook-filters'; + +const PG_INT_MAX = 2_147_483_647; +const CURSOR_MAX_LENGTH = 512; +const TIME_ZONE_MAX_LENGTH = 100; + +/** Query strings only ever carry text, so only the two spellings of a boolean are converted. */ +function toBoolean({ value }: TransformFnParams): unknown { + return value === 'true' ? true : value === 'false' ? false : value; +} + +/** `a,b` and a repeated `?x=a&x=b` both arrive as one de-duplicated list. */ +function toList({ value }: TransformFnParams): unknown { + const raw: unknown[] = Array.isArray(value) ? value : [value]; + if (!raw.every((part): part is string => typeof part === 'string')) return value; + const parts = raw + .flatMap((part) => part.split(',')) + .map((part) => part.trim()) + .filter(Boolean); + return [...new Set(parts)]; +} + +function trimmed({ value }: TransformFnParams): unknown { + return typeof value === 'string' ? value.trim() : value; +} + +function IsDateKey(validationOptions?: ValidationOptions): PropertyDecorator { + return ValidateBy( + { + name: 'isDateKey', + validator: { + validate: (value: unknown) => typeof value === 'string' && isDateKey(value), + defaultMessage: () => '$property must be a calendar date in YYYY-MM-DD form', + }, + }, + validationOptions, + ); +} + +function IsIanaTimeZone(validationOptions?: ValidationOptions): PropertyDecorator { + return ValidateBy( + { + name: 'isIanaTimeZone', + validator: { + validate: (value: unknown) => typeof value === 'string' && value.trim() !== '' && isValidTimeZone(value), + defaultMessage: () => '$property must be an IANA time zone', + }, + }, + validationOptions, + ); +} + +/** The filters every entry route shares. `false` on a boolean means no filter, as in the annotation hub. */ +export class NotebookFiltersQueryDto { + @IsOptional() + @Transform(toBoolean) + @IsBoolean() + starred?: boolean; + + /** Comma-separated stored colour strings. */ + @IsOptional() + @IsString() + @MaxLength(300) + colors?: string; + + @IsOptional() + @Transform(toBoolean) + @IsBoolean() + hasNote?: boolean; + + @IsOptional() + @Transform(toBoolean) + @IsBoolean() + approximate?: boolean; + + /** Comma-separated origins (web, koreader, kobo). */ + @IsOptional() + @Transform(toList) + @IsArray() + @ArrayMaxSize(NOTEBOOK_ORIGINS.length) + @IsIn(NOTEBOOK_ORIGINS, { each: true }) + origins?: NotebookOrigin[]; + + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(1) + @Max(PG_INT_MAX) + bookId?: number; + + @IsOptional() + @Transform(trimmed) + @IsString() + @MaxLength(NOTEBOOK_SEARCH_MAX_LENGTH) + q?: string; +} + +export class NotebookEntriesQueryDto extends NotebookFiltersQueryDto { + /** Comma-separated kinds; every kind when absent. */ + @IsOptional() + @Transform(toList) + @IsArray() + @ArrayMaxSize(NOTEBOOK_ENTRY_KINDS.length) + @IsIn(NOTEBOOK_ENTRY_KINDS, { each: true }) + kinds?: NotebookEntryKind[]; + + @IsOptional() + @IsIn(NOTEBOOK_SORTS) + sort?: NotebookSort; + + @IsOptional() + @IsString() + @MaxLength(CURSOR_MAX_LENGTH) + cursor?: string; + + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(1) + @Max(NOTEBOOK_PAGE_SIZE_MAX) + limit?: number; +} + +export class NotebookOverviewQueryDto extends NotebookFiltersQueryDto {} + +export class NotebookBooksQueryDto { + /** Matched against the book's title and its authors' names. */ + @IsOptional() + @Transform(trimmed) + @IsString() + @MaxLength(NOTEBOOK_SEARCH_MAX_LENGTH) + q?: string; + + @IsOptional() + @IsString() + @MaxLength(CURSOR_MAX_LENGTH) + cursor?: string; + + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(1) + @Max(NOTEBOOK_BOOKS_PAGE_SIZE_MAX) + limit?: number; +} + +/** Exactly one of `date` (the day's deck) and `seed` (a shuffle of the filtered highlights). */ +export class NotebookReviewQueryDto extends NotebookFiltersQueryDto { + @IsOptional() + @IsDateKey() + date?: string; + + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(Number.MIN_SAFE_INTEGER) + @Max(Number.MAX_SAFE_INTEGER) + seed?: number; +} + +export class NotebookOnThisDayQueryDto { + @IsDateKey() + date!: string; + + /** Defaults to the reader's timezone setting. */ + @IsOptional() + @IsString() + @MaxLength(TIME_ZONE_MAX_LENGTH) + @IsIanaTimeZone() + tz?: string; +} + +export class NotebookTrashQueryDto { + @IsOptional() + @IsString() + @MaxLength(CURSOR_MAX_LENGTH) + cursor?: string; + + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(1) + @Max(NOTEBOOK_PAGE_SIZE_MAX) + limit?: number; +} diff --git a/server/src/modules/notebook/notebook-cursor.test.ts b/server/src/modules/notebook/notebook-cursor.test.ts new file mode 100644 index 000000000..81b446b54 --- /dev/null +++ b/server/src/modules/notebook/notebook-cursor.test.ts @@ -0,0 +1,47 @@ +import { BadRequestException } from '@nestjs/common'; + +import { decodeNotebookCursor, encodeNotebookCursor, type NotebookCursor } from './notebook-cursor'; + +function rawCursor(value: unknown): string { + return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url'); +} + +describe('notebook cursor', () => { + const cursor: NotebookCursor = { scope: 'newest', key: '2026-03-04T05:06:07.123456Z', rank: 2, id: 981 }; + + it('round-trips every field, keeping the key to the microsecond', () => { + const encoded = encodeNotebookCursor(cursor); + expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/); + expect(decodeNotebookCursor(encoded, 'newest')).toEqual(cursor); + }); + + it.each(['oldest', 'edited', 'books', 'trash'] as const)('round-trips the %s scope', (scope) => { + expect(decodeNotebookCursor(encodeNotebookCursor({ ...cursor, scope }), scope)).toEqual({ ...cursor, scope }); + }); + + it('rejects a cursor issued for another sort', () => { + const encoded = encodeNotebookCursor(cursor); + expect(() => decodeNotebookCursor(encoded, 'oldest')).toThrow(BadRequestException); + expect(() => decodeNotebookCursor(encoded, 'oldest')).toThrow('Cursor belongs to a different listing or sort'); + expect(() => decodeNotebookCursor(encoded, 'trash')).toThrow(BadRequestException); + }); + + it.each([ + ['not base64url', 'abc+/='], + ['not JSON', Buffer.from('nope', 'utf8').toString('base64url')], + ['an array', rawCursor([1, 2])], + ['null', rawCursor(null)], + ['a missing key', rawCursor({ s: 'newest', k: 0, i: 1 })], + ['a numeric key', rawCursor({ s: 'newest', t: 1, k: 0, i: 1 })], + ['a millisecond key', rawCursor({ s: 'newest', t: '2026-03-04T05:06:07.123Z', k: 0, i: 1 })], + ['an impossible date', rawCursor({ s: 'newest', t: '2026-13-04T05:06:07.123456Z', k: 0, i: 1 })], + ['SQL in the key', rawCursor({ s: 'newest', t: "2026-03-04'; drop table x; --", k: 0, i: 1 })], + ['an unknown kind rank', rawCursor({ s: 'newest', t: cursor.key, k: 4, i: 1 })], + ['a fractional rank', rawCursor({ s: 'newest', t: cursor.key, k: 0.5, i: 1 })], + ['a zero id', rawCursor({ s: 'newest', t: cursor.key, k: 0, i: 0 })], + ['an id past int4', rawCursor({ s: 'newest', t: cursor.key, k: 0, i: 2_147_483_648 })], + ['a string id', rawCursor({ s: 'newest', t: cursor.key, k: 0, i: '5' })], + ])('rejects %s', (_label, raw) => { + expect(() => decodeNotebookCursor(raw, 'newest')).toThrow(BadRequestException); + }); +}); diff --git a/server/src/modules/notebook/notebook-cursor.ts b/server/src/modules/notebook/notebook-cursor.ts new file mode 100644 index 000000000..0f7d8e542 --- /dev/null +++ b/server/src/modules/notebook/notebook-cursor.ts @@ -0,0 +1,47 @@ +import { BadRequestException } from '@nestjs/common'; + +import { NOTEBOOK_ENTRY_KINDS, type NotebookSort } from '@bookorbit/types'; + +/** Which listing a cursor pages: an entry sort, the books list, or the trash. */ +export type NotebookCursorScope = NotebookSort | 'books' | 'trash'; + +/** + * A keyset position. `key` is the sort timestamp exactly as the database rendered it, to the + * microsecond, so the next page compares against the stored value and not a rounded copy of it. + */ +export interface NotebookCursor { + scope: NotebookCursorScope; + key: string; + rank: number; + id: number; +} + +const BASE64URL_RE = /^[A-Za-z0-9_-]+$/; +const SORT_KEY_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; +const PG_INT_MAX = 2_147_483_647; + +function invalid(): BadRequestException { + return new BadRequestException('Invalid cursor'); +} + +export function encodeNotebookCursor(cursor: NotebookCursor): string { + return Buffer.from(JSON.stringify({ s: cursor.scope, t: cursor.key, k: cursor.rank, i: cursor.id }), 'utf8').toString('base64url'); +} + +export function decodeNotebookCursor(raw: string, scope: NotebookCursorScope): NotebookCursor { + if (!BASE64URL_RE.test(raw)) throw invalid(); + let parsed: unknown; + try { + parsed = JSON.parse(Buffer.from(raw, 'base64url').toString('utf8')); + } catch { + throw invalid(); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) throw invalid(); + const { s, t, k, i } = parsed as Record; + if (typeof s !== 'string' || typeof t !== 'string' || typeof k !== 'number' || typeof i !== 'number') throw invalid(); + if (s !== scope) throw new BadRequestException('Cursor belongs to a different listing or sort'); + if (!SORT_KEY_RE.test(t) || Number.isNaN(Date.parse(t.slice(0, 23) + 'Z'))) throw invalid(); + if (!Number.isInteger(k) || k < 0 || k >= NOTEBOOK_ENTRY_KINDS.length) throw invalid(); + if (!Number.isInteger(i) || i < 1 || i > PG_INT_MAX) throw invalid(); + return { scope: s, key: t, rank: k, id: i }; +} diff --git a/server/src/modules/notebook/notebook-dates.test.ts b/server/src/modules/notebook/notebook-dates.test.ts new file mode 100644 index 000000000..ac5c14c70 --- /dev/null +++ b/server/src/modules/notebook/notebook-dates.test.ts @@ -0,0 +1,72 @@ +import { ON_THIS_DAY_YEARS_MAX, isoFromSortKey, localYearOfSortKey, onThisDayRanges, rangeYearOfSortKey, reviewOlderThan } from './notebook-dates'; + +describe('onThisDayRanges', () => { + it('is empty when the reader has no history', () => { + expect(onThisDayRanges('2026-10-04', null, 'UTC')).toEqual([]); + }); + + it('is empty when the oldest entry is from this year', () => { + expect(onThisDayRanges('2026-10-04', 2026, 'UTC')).toEqual([]); + }); + + it('covers each earlier year back to the first entry year, newest first, as local days', () => { + expect(onThisDayRanges('2026-10-04', 2024, 'Europe/Berlin')).toEqual([ + { year: 2025, start: '2025-10-03T22:00:00.000Z', end: '2025-10-04T22:00:00.000Z' }, + { year: 2024, start: '2024-10-03T22:00:00.000Z', end: '2024-10-04T22:00:00.000Z' }, + ]); + }); + + it('follows the zone across a daylight saving change', () => { + // 2025-03-30 is 23 hours long in Berlin. + expect(onThisDayRanges('2026-03-30', 2025, 'Europe/Berlin')).toEqual([ + { year: 2025, start: '2025-03-29T23:00:00.000Z', end: '2025-03-30T22:00:00.000Z' }, + ]); + expect(onThisDayRanges('2026-10-04', 2025, 'America/Los_Angeles')[0]).toEqual({ + year: 2025, + start: '2025-10-04T07:00:00.000Z', + end: '2025-10-05T07:00:00.000Z', + }); + }); + + it('skips years without 29 February', () => { + expect(onThisDayRanges('2028-02-29', 2019, 'UTC').map((range) => range.year)).toEqual([2024, 2020]); + }); + + it(`looks back at most ${ON_THIS_DAY_YEARS_MAX} years however old the oldest entry claims to be`, () => { + const ranges = onThisDayRanges('2026-10-04', 1, 'UTC'); + expect(ranges).toHaveLength(ON_THIS_DAY_YEARS_MAX); + expect(ranges.at(-1)?.year).toBe(2026 - ON_THIS_DAY_YEARS_MAX); + }); +}); + +describe('reviewOlderThan', () => { + it('is the UTC start of the day thirty days before the deck date', () => { + expect(reviewOlderThan('2026-10-04')).toBe('2026-09-04T00:00:00.000Z'); + expect(reviewOlderThan('2026-03-01')).toBe('2026-01-30T00:00:00.000Z'); + }); +}); + +describe('isoFromSortKey', () => { + it('drops the microseconds to the millisecond ISO form', () => { + expect(isoFromSortKey('2026-10-04T17:50:19.006736Z')).toBe('2026-10-04T17:50:19.006Z'); + }); +}); + +describe('localYearOfSortKey', () => { + it('reads the year in the reader zone', () => { + expect(localYearOfSortKey('2022-12-31T20:00:00.000000Z', 'Asia/Kolkata')).toBe(2023); + expect(localYearOfSortKey('2022-12-31T20:00:00.000000Z', 'UTC')).toBe(2022); + expect(localYearOfSortKey('2023-01-01T03:00:00.000000Z', 'America/Los_Angeles')).toBe(2022); + }); +}); + +describe('rangeYearOfSortKey', () => { + const ranges = onThisDayRanges('2026-10-04', 2024, 'Europe/Berlin'); + + it('finds the range a key falls in, half-open at the end', () => { + expect(rangeYearOfSortKey('2025-10-03T22:00:00.000000Z', ranges)).toBe(2025); + expect(rangeYearOfSortKey('2025-10-04T21:59:59.999999Z', ranges)).toBe(2025); + expect(rangeYearOfSortKey('2025-10-04T22:00:00.000000Z', ranges)).toBeNull(); + expect(rangeYearOfSortKey('2024-10-04T12:00:00.000000Z', ranges)).toBe(2024); + }); +}); diff --git a/server/src/modules/notebook/notebook-dates.ts b/server/src/modules/notebook/notebook-dates.ts new file mode 100644 index 000000000..9fba431d1 --- /dev/null +++ b/server/src/modules/notebook/notebook-dates.ts @@ -0,0 +1,71 @@ +import { addDateKeyDays } from '../../common/utils/reading-daily-stats.utils'; +import { getYearInTimeZone, isDateKey, toTimeZoneStartOfDay } from '../../common/utils/timezone.utils'; + +/** A deck's "older" tier is anything written at least this long before the deck's date. */ +export const REVIEW_OLDER_THAN_DAYS = 30; +/** On this day looks back at most this many years, whatever the oldest entry claims. */ +export const ON_THIS_DAY_YEARS_MAX = 100; +/** Date keys below this year are outside what the calendar helpers handle; no real entry is that old. */ +const EARLIEST_YEAR = 1000; + +export interface OnThisDayRange { + year: number; + start: string; + end: string; +} + +/** + * One half-open range per earlier year, newest first: that year's month and day, from local + * midnight to the next local midnight in `timeZone`. A year without the day (29 February outside a + * leap year) has no range, and neither has a day whose local midnight the zone skips. + */ +export function onThisDayRanges(date: string, firstYear: number | null, timeZone: string): OnThisDayRange[] { + if (firstYear == null) return []; + const year = Number(date.slice(0, 4)); + const monthDay = date.slice(4); + const earliest = Math.max(firstYear, year - ON_THIS_DAY_YEARS_MAX, EARLIEST_YEAR); + const ranges: OnThisDayRange[] = []; + for (let candidate = year - 1; candidate >= earliest; candidate--) { + const day = `${String(candidate).padStart(4, '0')}${monthDay}`; + if (!isDateKey(day)) continue; + try { + ranges.push({ + year: candidate, + start: toTimeZoneStartOfDay(day, timeZone).toISOString(), + end: toTimeZoneStartOfDay(addDateKeyDays(day, 1), timeZone).toISOString(), + }); + } catch (error) { + if (!(error instanceof RangeError)) throw error; + } + } + return ranges; +} + +/** The start of the day `REVIEW_OLDER_THAN_DAYS` before `date`, in UTC, so a deck never depends on the clock. */ +export function reviewOlderThan(date: string): string { + return toTimeZoneStartOfDay(addDateKeyDays(date, -REVIEW_OLDER_THAN_DAYS), 'UTC').toISOString(); +} + +/** A microsecond sort key as the millisecond ISO string every other timestamp in the API uses. */ +export function isoFromSortKey(key: string): string { + return new Date(`${key.slice(0, 23)}Z`).toISOString(); +} + +/** The year a sort key falls in for `timeZone`, or its UTC year when the calendar helpers cannot place it. */ +export function localYearOfSortKey(key: string, timeZone: string): number { + try { + return getYearInTimeZone(new Date(isoFromSortKey(key)), timeZone); + } catch (error) { + if (!(error instanceof RangeError)) throw error; + return Number(key.slice(0, 4)); + } +} + +/** + * The year of the range a sort key falls in, or null when it is in none. Ranges start on whole + * milliseconds, so dropping the key's microseconds can never move it across a boundary. + */ +export function rangeYearOfSortKey(key: string, ranges: readonly OnThisDayRange[]): number | null { + const at = Date.parse(isoFromSortKey(key)); + return ranges.find((range) => at >= Date.parse(range.start) && at < Date.parse(range.end))?.year ?? null; +} diff --git a/server/src/modules/notebook/notebook-filters.test.ts b/server/src/modules/notebook/notebook-filters.test.ts new file mode 100644 index 000000000..b6f1242e0 --- /dev/null +++ b/server/src/modules/notebook/notebook-filters.test.ts @@ -0,0 +1,74 @@ +import { NOTEBOOK_ENTRY_KINDS } from '@bookorbit/types'; + +import { NOTEBOOK_KIND_RANK, hasHighlightOnlyFilter, notebookKindForRank, resolveNotebookKinds, toNotebookFilters } from './notebook-filters'; + +describe('toNotebookFilters', () => { + it('treats false booleans as no filter', () => { + expect(toNotebookFilters({ starred: false, hasNote: false, approximate: false })).toEqual({}); + }); + + it('keeps true booleans', () => { + expect(toNotebookFilters({ starred: true, hasNote: true, approximate: true })).toEqual({ starred: true, hasNote: true, approximate: true }); + }); + + it('splits, trims and de-duplicates the colour list', () => { + expect(toNotebookFilters({ colors: ' #FACC15, yellow,,#FACC15 ' }).colors).toEqual(['#FACC15', 'yellow']); + expect(toNotebookFilters({ colors: ' , ' }).colors).toBeUndefined(); + }); + + it('keeps origins and the book id', () => { + expect(toNotebookFilters({ origins: ['kobo', 'kobo', 'web'], bookId: 12 })).toEqual({ origins: ['kobo', 'web'], bookId: 12 }); + expect(toNotebookFilters({ origins: [] }).origins).toBeUndefined(); + }); + + it('turns q into an escaped LIKE pattern', () => { + expect(toNotebookFilters({ q: ' 50%_off\\now ' }).searchPattern).toBe('%50\\%\\_off\\\\now%'); + }); + + it('ignores a blank q', () => { + expect(toNotebookFilters({ q: ' ' }).searchPattern).toBeUndefined(); + }); +}); + +describe('resolveNotebookKinds', () => { + it('returns every kind by default, in rank order', () => { + expect(resolveNotebookKinds(undefined, {})).toEqual([...NOTEBOOK_ENTRY_KINDS]); + expect(resolveNotebookKinds([], {})).toEqual([...NOTEBOOK_ENTRY_KINDS]); + }); + + it('honours the requested kinds', () => { + expect(resolveNotebookKinds(['review', 'journal'], {})).toEqual(['journal', 'review']); + }); + + it.each([[{ starred: true as const }], [{ colors: ['yellow'] }], [{ approximate: true as const }]])( + 'leaves only highlights while a highlight-only filter is set: %j', + (filters) => { + expect(hasHighlightOnlyFilter(filters)).toBe(true); + expect(resolveNotebookKinds(undefined, filters)).toEqual(['highlight']); + expect(resolveNotebookKinds(['journal', 'bookmark'], filters)).toEqual([]); + }, + ); + + it('keeps every kind for hasNote, which journal entries and reviews always meet', () => { + expect(resolveNotebookKinds(undefined, { hasNote: true })).toEqual([...NOTEBOOK_ENTRY_KINDS]); + }); + + it('drops journal entries and reviews for an origin filter without web', () => { + expect(resolveNotebookKinds(undefined, { origins: ['koreader', 'kobo'] })).toEqual(['highlight', 'bookmark']); + }); + + it('keeps journal entries and reviews when web is among the origins', () => { + expect(resolveNotebookKinds(undefined, { origins: ['web', 'kobo'] })).toEqual([...NOTEBOOK_ENTRY_KINDS]); + }); +}); + +describe('kind ranks', () => { + it('maps every rank back to its kind', () => { + for (const kind of NOTEBOOK_ENTRY_KINDS) expect(notebookKindForRank(NOTEBOOK_KIND_RANK[kind])).toBe(kind); + expect(NOTEBOOK_KIND_RANK).toEqual({ highlight: 0, journal: 1, bookmark: 2, review: 3 }); + }); + + it('throws on an unknown rank', () => { + expect(() => notebookKindForRank(9)).toThrow(RangeError); + }); +}); diff --git a/server/src/modules/notebook/notebook-filters.ts b/server/src/modules/notebook/notebook-filters.ts new file mode 100644 index 000000000..5c2999258 --- /dev/null +++ b/server/src/modules/notebook/notebook-filters.ts @@ -0,0 +1,91 @@ +import { NOTEBOOK_ENTRY_KINDS, type ContentFilterRules, type NotebookEntryKind } from '@bookorbit/types'; + +import { buildSearchPattern } from '../../common/utils/accent-insensitive-search.utils'; + +export const NOTEBOOK_ORIGINS = ['web', 'koreader', 'kobo'] as const; +export type NotebookOrigin = (typeof NOTEBOOK_ORIGINS)[number]; + +/** Ties between entries written in the same microsecond break on kind, in this order. */ +export const NOTEBOOK_KIND_RANK: Record = { highlight: 0, journal: 1, bookmark: 2, review: 3 }; + +export function notebookKindForRank(rank: number): NotebookEntryKind { + const kind = NOTEBOOK_ENTRY_KINDS.find((candidate) => NOTEBOOK_KIND_RANK[candidate] === rank); + if (!kind) throw new RangeError(`Unknown notebook kind rank ${rank}`); + return kind; +} + +/** What a reader may see: the libraries they can open, narrowed by any content rules on their account. */ +export interface NotebookScope { + userId: number; + libraryIds: number[]; + contentFilters?: ContentFilterRules; +} + +/** The parsed entry filters. Every field is absent rather than false when it does not filter. */ +export interface NotebookEntryFilters { + starred?: true; + colors?: string[]; + hasNote?: true; + approximate?: true; + origins?: NotebookOrigin[]; + bookId?: number; + /** A ready LIKE pattern: wrapped in `%` with `%`, `_` and `\` in the term escaped. */ + searchPattern?: string; +} + +export interface NotebookFiltersInput { + starred?: boolean; + colors?: string; + hasNote?: boolean; + approximate?: boolean; + origins?: NotebookOrigin[]; + bookId?: number; + q?: string; +} + +function splitList(value: string | undefined): string[] | undefined { + if (!value) return undefined; + const parts = [ + ...new Set( + value + .split(',') + .map((part) => part.trim()) + .filter(Boolean), + ), + ]; + return parts.length > 0 ? parts : undefined; +} + +export function toNotebookFilters(input: NotebookFiltersInput): NotebookEntryFilters { + const term = input.q?.trim(); + return { + ...(input.starred && { starred: true as const }), + ...(splitList(input.colors) && { colors: splitList(input.colors) }), + ...(input.hasNote && { hasNote: true as const }), + ...(input.approximate && { approximate: true as const }), + ...(input.origins && input.origins.length > 0 && { origins: [...new Set(input.origins)] }), + ...(input.bookId !== undefined && { bookId: input.bookId }), + ...(term && { searchPattern: buildSearchPattern(term) }), + }; +} + +/** Highlight-only filters leave every other kind out while set. */ +export function hasHighlightOnlyFilter(filters: NotebookEntryFilters): boolean { + return filters.starred === true || (filters.colors?.length ?? 0) > 0 || filters.approximate === true; +} + +/** + * The kinds a request can return, in rank order. Journal entries and reviews have no origin of + * their own and count as `web`, so an origin filter without `web` leaves them out. + */ +export function resolveNotebookKinds(requested: readonly NotebookEntryKind[] | undefined, filters: NotebookEntryFilters): NotebookEntryKind[] { + const wanted = new Set(requested && requested.length > 0 ? requested : NOTEBOOK_ENTRY_KINDS); + const highlightOnly = hasHighlightOnlyFilter(filters); + const webOnlyKindsAllowed = !filters.origins || filters.origins.includes('web'); + return NOTEBOOK_ENTRY_KINDS.filter((kind) => { + if (!wanted.has(kind)) return false; + if (highlightOnly && kind !== 'highlight') return false; + if ((kind === 'journal' || kind === 'review') && !webOnlyKindsAllowed) return false; + return true; + }); +} diff --git a/server/src/modules/notebook/notebook-mapper.test.ts b/server/src/modules/notebook/notebook-mapper.test.ts new file mode 100644 index 000000000..907e7428c --- /dev/null +++ b/server/src/modules/notebook/notebook-mapper.test.ts @@ -0,0 +1,190 @@ +import { + toNotebookBookLabel, + toNotebookBookmark, + toNotebookHighlight, + toNotebookJournal, + toNotebookReview, + type NotebookHighlightRow, + type NotebookLabelRow, +} from './notebook-mapper'; + +const created = new Date('2026-01-10T00:00:00.000Z'); +const updated = new Date('2026-01-12T08:30:00.000Z'); + +function highlightRow(overrides: Partial = {}): NotebookHighlightRow { + return { + id: 4, + bookId: 9, + text: 'A passage', + color: '#FACC15', + style: 'highlight', + note: null, + chapterTitle: 'One', + origin: 'kobo', + sourceCreatedAt: null, + createdAt: created, + updatedAt: updated, + deletedAt: null, + starredAt: null, + cfi: 'epubcfi(/6/4!/4/2/1:0)', + cfiStatus: 'repaired', + ...overrides, + }; +} + +describe('toNotebookHighlight', () => { + it('maps the hub fields and dates it by when it was highlighted', () => { + const backdated = new Date('2025-12-31T23:59:59.000Z'); + expect(toNotebookHighlight(highlightRow({ sourceCreatedAt: backdated, starredAt: updated }))).toEqual({ + kind: 'highlight', + id: 4, + bookId: 9, + clientId: null, + createdAt: '2025-12-31T23:59:59.000Z', + updatedAt: '2026-01-12T08:30:00.000Z', + text: 'A passage', + color: '#FACC15', + style: 'highlight', + note: null, + chapterTitle: 'One', + cfi: 'epubcfi(/6/4!/4/2/1:0)', + positionStatus: 'repaired', + origin: 'kobo', + starredAt: '2026-01-12T08:30:00.000Z', + }); + }); + + it('reports no position status without a CFI row and leaves deletedAt out of active listings', () => { + const item = toNotebookHighlight(highlightRow({ cfi: null, cfiStatus: null })); + expect(item.positionStatus).toBeNull(); + expect(item.cfi).toBeNull(); + expect(item).not.toHaveProperty('deletedAt'); + }); + + it('reads a CFI without a status as exact', () => { + expect(toNotebookHighlight(highlightRow({ cfiStatus: null })).positionStatus).toBe('exact'); + }); + + it('carries deletedAt for the trash', () => { + expect(toNotebookHighlight(highlightRow({ deletedAt: updated }), { withDeletedAt: true }).deletedAt).toBe('2026-01-12T08:30:00.000Z'); + }); +}); + +describe('toNotebookJournal', () => { + it('maps the journal entry fields', () => { + const row = { + id: 3, + clientId: '4f9c7a52-3d1e-4b6a-9a51-0c2f6e8d7b13', + bookId: 9, + body: 'Thought', + quote: null, + chapterTitle: null, + positionPercent: 12.5, + cfi: null, + positionSeconds: null, + createdAt: created, + updatedAt: updated, + deletedAt: null, + }; + expect(toNotebookJournal(row)).toEqual({ + kind: 'journal', + id: 3, + bookId: 9, + clientId: row.clientId, + createdAt: created.toISOString(), + updatedAt: updated.toISOString(), + body: 'Thought', + quote: null, + chapterTitle: null, + positionPercent: 12.5, + cfi: null, + positionSeconds: null, + }); + expect(toNotebookJournal({ ...row, deletedAt: updated }, { withDeletedAt: true }).deletedAt).toBe(updated.toISOString()); + }); +}); + +describe('toNotebookBookmark', () => { + it('maps an audiobook bookmark with its time', () => { + expect( + toNotebookBookmark({ + id: 8, + clientId: '8a1c3b0e-0f43-4d61-9d7a-3b6f5a2c9e10', + bookId: 2, + title: 'Chapter 3', + note: null, + cfi: null, + positionSeconds: 1234.5, + origin: 'web', + createdAt: created, + updatedAt: updated, + deletedAt: null, + }), + ).toEqual({ + kind: 'bookmark', + id: 8, + bookId: 2, + clientId: '8a1c3b0e-0f43-4d61-9d7a-3b6f5a2c9e10', + createdAt: created.toISOString(), + updatedAt: updated.toISOString(), + title: 'Chapter 3', + note: null, + cfi: null, + positionSeconds: 1234.5, + origin: 'web', + }); + }); +}); + +describe('toNotebookReview', () => { + it('uses the book id as its id and the edit time as both dates', () => { + expect(toNotebookReview({ bookId: 5, note: 'Loved it', updatedAt: updated, rating: 4 })).toEqual({ + kind: 'review', + id: 5, + bookId: 5, + clientId: null, + createdAt: updated.toISOString(), + updatedAt: updated.toISOString(), + body: 'Loved it', + rating: 4, + }); + expect(toNotebookReview({ bookId: 5, note: 'x', updatedAt: updated, rating: null }).rating).toBeNull(); + }); +}); + +describe('toNotebookBookLabel', () => { + const row: NotebookLabelRow = { + id: 5, + title: 'Dune', + author: 'Frank Herbert, Brian Herbert', + coverSource: 'extracted', + coverAspectRatio: '1/1', + updatedAt: updated, + readFileId: 77, + readFileFormat: 'EPUB', + hasAudio: true, + }; + + it('maps the label with the given cover version', () => { + expect(toNotebookBookLabel(row, 'audio:x:-')).toEqual({ + id: 5, + title: 'Dune', + author: 'Frank Herbert, Brian Herbert', + hasCover: true, + coverVersion: 'audio:x:-', + coverAspectRatio: '1/1', + readFileId: 77, + readFileFormat: 'epub', + hasAudio: true, + }); + }); + + it('falls back to the portrait ratio and reports a book with nothing to read', () => { + expect( + toNotebookBookLabel( + { ...row, coverSource: null, coverAspectRatio: 'odd', readFileId: null, readFileFormat: null, hasAudio: false }, + 'legacy:x', + ), + ).toMatchObject({ hasCover: false, coverAspectRatio: '2/3', readFileId: null, readFileFormat: null, hasAudio: false }); + }); +}); diff --git a/server/src/modules/notebook/notebook-mapper.ts b/server/src/modules/notebook/notebook-mapper.ts new file mode 100644 index 000000000..7cdf1f4ef --- /dev/null +++ b/server/src/modules/notebook/notebook-mapper.ts @@ -0,0 +1,163 @@ +import { + normalizeCoverAspectRatio, + type AnnotationPositionStatus, + type NotebookBookLabel, + type NotebookBookmarkItem, + type NotebookHighlightEntry, + type NotebookJournalItem, + type NotebookReviewItem, +} from '@bookorbit/types'; + +import type { AnnotationRow, BookJournalEntryRow, BookmarkRow } from '../../db/schema'; + +export type NotebookHighlightRow = Pick< + AnnotationRow, + | 'id' + | 'bookId' + | 'text' + | 'color' + | 'style' + | 'note' + | 'chapterTitle' + | 'origin' + | 'sourceCreatedAt' + | 'createdAt' + | 'updatedAt' + | 'deletedAt' + | 'starredAt' +> & { cfi: string | null; cfiStatus: string | null }; + +export type NotebookJournalRow = Pick< + BookJournalEntryRow, + | 'id' + | 'clientId' + | 'bookId' + | 'body' + | 'quote' + | 'chapterTitle' + | 'positionPercent' + | 'cfi' + | 'positionSeconds' + | 'createdAt' + | 'updatedAt' + | 'deletedAt' +>; + +export type NotebookBookmarkRow = Pick< + BookmarkRow, + 'id' | 'clientId' | 'bookId' | 'title' | 'note' | 'cfi' | 'positionSeconds' | 'origin' | 'createdAt' | 'updatedAt' | 'deletedAt' +>; + +export interface NotebookReviewRow { + bookId: number; + note: string | null; + updatedAt: Date; + rating: number | null; +} + +export interface NotebookLabelRow { + id: number; + title: string | null; + author: string | null; + coverSource: string | null; + coverAspectRatio: string; + updatedAt: Date; + readFileId: number | null; + readFileFormat: string | null; + hasAudio: boolean; +} + +/** Trash items carry `deletedAt`; every other listing leaves the field out. */ +export interface NotebookMapOptions { + withDeletedAt?: boolean; +} + +function deletedAtField(deletedAt: Date | null, options: NotebookMapOptions): { deletedAt?: string | null } { + return options.withDeletedAt ? { deletedAt: deletedAt ? deletedAt.toISOString() : null } : {}; +} + +/** Mirrors the annotation hub's item: the write time a device reported wins, and a CFI row without a status reads as exact. */ +export function toNotebookHighlight(row: NotebookHighlightRow, options: NotebookMapOptions = {}): NotebookHighlightEntry { + return { + kind: 'highlight', + id: row.id, + bookId: row.bookId, + clientId: null, + createdAt: (row.sourceCreatedAt ?? row.createdAt).toISOString(), + updatedAt: row.updatedAt.toISOString(), + ...deletedAtField(row.deletedAt, options), + text: row.text, + color: row.color, + style: row.style, + note: row.note ?? null, + chapterTitle: row.chapterTitle ?? null, + cfi: row.cfi ?? null, + positionStatus: (row.cfi != null || row.cfiStatus != null ? (row.cfiStatus ?? 'exact') : null) as AnnotationPositionStatus | null, + origin: row.origin, + starredAt: row.starredAt ? row.starredAt.toISOString() : null, + }; +} + +export function toNotebookJournal(row: NotebookJournalRow, options: NotebookMapOptions = {}): NotebookJournalItem { + return { + kind: 'journal', + id: row.id, + bookId: row.bookId, + clientId: row.clientId, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + ...deletedAtField(row.deletedAt, options), + body: row.body, + quote: row.quote ?? null, + chapterTitle: row.chapterTitle ?? null, + positionPercent: row.positionPercent ?? null, + cfi: row.cfi ?? null, + positionSeconds: row.positionSeconds ?? null, + }; +} + +export function toNotebookBookmark(row: NotebookBookmarkRow): NotebookBookmarkItem { + return { + kind: 'bookmark', + id: row.id, + bookId: row.bookId, + clientId: row.clientId, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + title: row.title, + note: row.note ?? null, + cfi: row.cfi ?? null, + positionSeconds: row.positionSeconds ?? null, + origin: row.origin, + }; +} + +/** The server keeps only a review's latest edit time, so it is both when it was written and when it changed. */ +export function toNotebookReview(row: NotebookReviewRow): NotebookReviewItem { + const at = row.updatedAt.toISOString(); + return { + kind: 'review', + id: row.bookId, + bookId: row.bookId, + clientId: null, + createdAt: at, + updatedAt: at, + body: row.note ?? '', + rating: row.rating ?? null, + }; +} + +/** `coverVersion` is computed by the cover store, exactly as it is for a library book card. */ +export function toNotebookBookLabel(row: NotebookLabelRow, coverVersion: string): NotebookBookLabel { + return { + id: row.id, + title: row.title ?? null, + author: row.author ?? null, + hasCover: row.coverSource != null, + coverVersion, + coverAspectRatio: normalizeCoverAspectRatio(row.coverAspectRatio), + readFileId: row.readFileId ?? null, + readFileFormat: row.readFileFormat ? row.readFileFormat.toLowerCase() : null, + hasAudio: row.hasAudio === true, + }; +} diff --git a/server/src/modules/notebook/notebook-sql.test.ts b/server/src/modules/notebook/notebook-sql.test.ts new file mode 100644 index 000000000..94a9fbd98 --- /dev/null +++ b/server/src/modules/notebook/notebook-sql.test.ts @@ -0,0 +1,236 @@ +import { and, type SQL } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/node-postgres'; +import { PgDialect } from 'drizzle-orm/pg-core'; + +import { EMPTY_CONTENT_FILTER_RULES } from '@bookorbit/types'; + +import * as schema from '../../db/schema'; +import type { NotebookCursor } from './notebook-cursor'; +import { toNotebookFilters, type NotebookScope } from './notebook-filters'; +import { + NOTEBOOK_SOURCES, + armWhereSql, + bookActivitySql, + cursorSql, + dailyReviewSql, + filterSql, + inRangesSql, + keysetUnionSql, + overviewSql, + seededReviewSql, + sortColumnSql, + statusSql, +} from './notebook-sql'; + +const db = drizzle.mock({ schema }); +const dialect = new PgDialect(); +const scope: NotebookScope = { userId: 7, libraryIds: [1, 2] }; + +function render(fragment: SQL | SQL[]) { + const query = dialect.sqlToQuery(Array.isArray(fragment) ? (and(...fragment) ?? (fragment[0] as SQL)) : fragment); + return { sql: query.sql.replace(/\s+/g, ' '), params: query.params }; +} + +describe('filterSql', () => { + it('rules other kinds out while a highlight-only filter is set', () => { + for (const kind of ['journal', 'bookmark', 'review'] as const) { + expect(render(filterSql(NOTEBOOK_SOURCES[kind], { starred: true })).sql).toContain('false'); + } + const highlight = render(filterSql(NOTEBOOK_SOURCES.highlight, { starred: true, colors: ['#FACC15', 'yellow'] })); + expect(highlight.sql).toContain('"annotations"."starred_at" is not null'); + expect(highlight.sql).toContain('"annotations"."color" in ($1, $2)'); + expect(highlight.params).toEqual(['#FACC15', 'yellow']); + }); + + it('asks highlights and bookmarks for a non-blank note and lets journal entries and reviews through', () => { + expect(render(filterSql(NOTEBOOK_SOURCES.highlight, { hasNote: true })).sql).toBe(`"annotations"."note" ~ '[^[:space:]]'`); + expect(render(filterSql(NOTEBOOK_SOURCES.bookmark, { hasNote: true })).sql).toBe(`"bookmarks"."note" ~ '[^[:space:]]'`); + expect(filterSql(NOTEBOOK_SOURCES.journal, { hasNote: true })).toEqual([]); + expect(filterSql(NOTEBOOK_SOURCES.review, { hasNote: true })).toEqual([]); + }); + + it('filters origins on highlights and bookmarks and treats journal entries and reviews as web', () => { + const highlight = render(filterSql(NOTEBOOK_SOURCES.highlight, { origins: ['kobo'] })); + expect(highlight.sql).toBe('"annotations"."origin" in ($1)'); + expect(highlight.params).toEqual(['kobo']); + expect(render(filterSql(NOTEBOOK_SOURCES.bookmark, { origins: ['koreader'] })).sql).toBe('"bookmarks"."origin" in ($1)'); + expect(render(filterSql(NOTEBOOK_SOURCES.journal, { origins: ['kobo'] })).sql).toBe('false'); + expect(filterSql(NOTEBOOK_SOURCES.review, { origins: ['web', 'kobo'] })).toEqual([]); + }); + + it('counts a highlight as approximate unless an exact, non-empty CFI anchors it', () => { + const { sql } = render(filterSql(NOTEBOOK_SOURCES.highlight, { approximate: true })); + expect(sql).toContain('not exists'); + expect(sql).toContain(`"annotation_positions"."format" = 'cfi'`); + expect(sql).toContain(`"annotation_positions"."status" = 'exact'`); + expect(sql).toContain(`"annotation_positions"."pos0" <> ''`); + }); + + it('matches q against the entry text, the book title and its authors, with the pattern escaped and bound', () => { + const { searchPattern } = toNotebookFilters({ q: '50%_off' }); + const { sql, params } = render(filterSql(NOTEBOOK_SOURCES.journal, { searchPattern })); + expect(sql).toContain('public.bookorbit_unaccent("book_journal_entries"."body") ILIKE public.bookorbit_unaccent($1)'); + expect(sql).toContain('public.bookorbit_unaccent("book_journal_entries"."quote") ILIKE'); + expect(sql).toContain('public.bookorbit_unaccent("book_metadata"."title") ILIKE'); + expect(sql).toContain('public.bookorbit_unaccent("authors"."name") ILIKE'); + expect(sql).toContain('"book_authors"."book_id" = "book_journal_entries"."book_id"'); + expect(sql).not.toContain('50%_off'); + expect(params.every((param) => param === '%50\\%\\_off%')).toBe(true); + }); + + it('pins the book id on every kind, using the book id as the review id', () => { + expect(render(filterSql(NOTEBOOK_SOURCES.review, { bookId: 4 })).sql).toBe('"user_book_notes"."book_id" = $1'); + }); +}); + +describe('statusSql', () => { + it('splits active and trashed rows on deleted_at, and gives reviews no trash', () => { + expect(render(statusSql(NOTEBOOK_SOURCES.highlight, 'trashed')).sql).toBe('"annotations"."deleted_at" is not null'); + expect(render(statusSql(NOTEBOOK_SOURCES.bookmark, 'active')).sql).toBe('"bookmarks"."deleted_at" is null'); + expect(statusSql(NOTEBOOK_SOURCES.review, 'active')).toEqual([]); + expect(render(statusSql(NOTEBOOK_SOURCES.review, 'trashed')).sql).toBe('false'); + }); +}); + +describe('armWhereSql', () => { + it('scopes every arm to the user, the accessible libraries and the content rules', () => { + const { sql, params } = render( + armWhereSql(NOTEBOOK_SOURCES.bookmark, { ...scope, contentFilters: { ...EMPTY_CONTENT_FILTER_RULES, excludeTagIds: [9] } }, db, { + status: 'active', + }), + ); + expect(sql).toContain('"bookmarks"."user_id" = $1'); + expect(sql).toContain('"books"."library_id" in ($2, $3)'); + expect(sql).toContain('not exists (select 1 from "book_tags" where ("book_tags"."book_id" = "books"."id"'); + expect(params).toEqual([7, 1, 2, 9]); + }); + + it('only counts a review whose text is not blank', () => { + expect(render(armWhereSql(NOTEBOOK_SOURCES.review, scope, db, { status: 'active' })).sql).toContain(`"user_book_notes"."note" ~ '[^[:space:]]'`); + }); +}); + +describe('cursorSql', () => { + const ts = sortColumnSql(NOTEBOOK_SOURCES.journal, 'written'); + const cursor: NotebookCursor = { scope: 'newest', key: '2026-01-02T03:04:05.678901Z', rank: 1, id: 30 }; + + it('descends: earlier kinds may tie on time, the same kind compares (time, id), later kinds must be older', () => { + expect(render(cursorSql(NOTEBOOK_SOURCES.highlight, NOTEBOOK_SOURCES.highlight.writtenAt, cursor, 'desc')).sql).toBe( + 'coalesce("annotations"."source_created_at", "annotations"."created_at") <= $1::timestamptz', + ); + const same = render(cursorSql(NOTEBOOK_SOURCES.journal, ts, cursor, 'desc')); + expect(same.sql).toBe('("book_journal_entries"."created_at", "book_journal_entries"."id") < ($1::timestamptz, $2::int)'); + expect(same.params).toEqual(['2026-01-02T03:04:05.678901Z', 30]); + expect(render(cursorSql(NOTEBOOK_SOURCES.bookmark, NOTEBOOK_SOURCES.bookmark.writtenAt, cursor, 'desc')).sql).toBe( + '"bookmarks"."created_at" < $1::timestamptz', + ); + }); + + it('mirrors every bound when ascending', () => { + expect(render(cursorSql(NOTEBOOK_SOURCES.highlight, NOTEBOOK_SOURCES.highlight.writtenAt, cursor, 'asc')).sql).toContain('> $1::timestamptz'); + expect(render(cursorSql(NOTEBOOK_SOURCES.journal, ts, cursor, 'asc')).sql).toContain(') > ($1::timestamptz, $2::int)'); + expect(render(cursorSql(NOTEBOOK_SOURCES.review, NOTEBOOK_SOURCES.review.writtenAt, cursor, 'asc')).sql).toBe( + '"user_book_notes"."updated_at" >= $1::timestamptz', + ); + }); +}); + +describe('sortColumnSql', () => { + it('uses the write time, the edit time or the deletion time', () => { + expect(render(sortColumnSql(NOTEBOOK_SOURCES.highlight, 'edited')).sql).toBe('"annotations"."updated_at"'); + expect(render(sortColumnSql(NOTEBOOK_SOURCES.journal, 'deleted')).sql).toBe('"book_journal_entries"."deleted_at"'); + expect(render(sortColumnSql(NOTEBOOK_SOURCES.review, 'written')).sql).toBe('"user_book_notes"."updated_at"'); + expect(() => sortColumnSql(NOTEBOOK_SOURCES.review, 'deleted')).toThrow(RangeError); + }); +}); + +describe('keysetUnionSql', () => { + const arms = (['highlight', 'bookmark'] as const).map((kind) => ({ + source: NOTEBOOK_SOURCES[kind], + sortTs: NOTEBOOK_SOURCES[kind].writtenAt, + where: armWhereSql(NOTEBOOK_SOURCES[kind], scope, db, { status: 'active' }), + })); + + it('limits each arm on the index order before the merge and renders the key to the microsecond', () => { + const { sql } = render(keysetUnionSql(arms, { direction: 'desc', limit: 41 })); + expect(sql).toContain('order by coalesce("annotations"."source_created_at", "annotations"."created_at") desc, "annotations"."id" desc limit $'); + expect(sql).toContain('order by "bookmarks"."created_at" desc, "bookmarks"."id" desc limit $'); + expect(sql).toContain(' union all '); + expect(sql).toContain(`to_char(sort_ts at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') as sort_key`); + expect(sql).toContain('order by sort_ts desc, kind_rank desc, id desc limit $'); + }); + + it('adds the cursor bound to every arm', () => { + const cursor: NotebookCursor = { scope: 'oldest', key: '2026-01-02T03:04:05.000000Z', rank: 0, id: 3 }; + const { sql } = render(keysetUnionSql(arms, { direction: 'asc', limit: 5, cursor })); + expect(sql).toContain('"annotations"."created_at"), "annotations"."id") > ($'); + expect(sql).toContain('"bookmarks"."created_at" >= '); + expect(sql).toContain('order by sort_ts asc, kind_rank asc, id asc'); + }); + + it('never hands a time zone name to Postgres', () => { + expect(render(keysetUnionSql(arms, { direction: 'desc', limit: 20 })).sql).not.toMatch(/at time zone \$/); + }); +}); + +describe('inRangesSql', () => { + it('is false with no ranges and a half-open OR of ranges otherwise', () => { + expect(render(inRangesSql(NOTEBOOK_SOURCES.bookmark.writtenAt, [])).sql).toBe('false'); + const { sql, params } = render( + inRangesSql(NOTEBOOK_SOURCES.bookmark.writtenAt, [ + { start: '2025-10-03T22:00:00.000Z', end: '2025-10-04T22:00:00.000Z' }, + { start: '2024-10-03T22:00:00.000Z', end: '2024-10-04T22:00:00.000Z' }, + ]), + ); + expect(sql).toBe( + '(("bookmarks"."created_at" >= $1::timestamptz and "bookmarks"."created_at" < $2::timestamptz) or ("bookmarks"."created_at" >= $3::timestamptz and "bookmarks"."created_at" < $4::timestamptz))', + ); + expect(params).toHaveLength(4); + }); +}); + +describe('overviewSql', () => { + it('counts every kind in one statement and zeroes a kind the filters leave out', () => { + const { sql } = render(overviewSql(scope, db, { starred: true }, ['highlight'])); + for (const name of ['highlights', 'journal', 'bookmarks', 'reviews', 'starred', 'with_notes', 'approximate', 'books', 'colors', 'origins']) { + expect(sql).toContain(` as ${name}`); + } + const journalCte = sql.slice(sql.indexOf('j as ('), sql.indexOf('b as (')); + expect(journalCte).toContain('and false'); + }); +}); + +describe('bookActivitySql', () => { + it('folds activity per book, scopes on books and pages by (last activity, book id)', () => { + const cursor: NotebookCursor = { scope: 'books', key: '2026-01-02T03:04:05.000000Z', rank: 0, id: 12 }; + const { sql } = render(bookActivitySql(scope, db, { limit: 31, cursor, searchPattern: '%tolk%' })); + expect(sql).toContain('group by book_id'); + expect(sql).toContain('bool_or(kind_rank = $'); + expect(sql).toContain('"books"."library_id" in ('); + expect(sql).toContain('(activity.last_ts, activity.book_id) < ('); + expect(sql).toContain('"book_metadata"."book_id" = "books"."id"'); + expect(sql).toContain('order by activity.last_ts desc, activity.book_id desc'); + }); +}); + +describe('review decks', () => { + it('orders the daily deck by tier, then a hash of user, date and id, one per book', () => { + const first = render(dailyReviewSql(scope, db, { date: '2026-10-04', olderThan: '2026-09-04T00:00:00.000Z', limit: 10 })); + const again = render(dailyReviewSql(scope, db, { date: '2026-10-04', olderThan: '2026-09-04T00:00:00.000Z', limit: 10 })); + const nextDay = render(dailyReviewSql(scope, db, { date: '2026-10-05', olderThan: '2026-09-05T00:00:00.000Z', limit: 10 })); + expect(first).toEqual(again); + expect(nextDay.params).not.toEqual(first.params); + expect(first.sql).toContain('select distinct on ("annotations"."book_id")'); + expect(first.sql).toContain(`md5($`); + expect(first.sql).toContain(`"annotations"."text" ~ '[^[:space:]]'`); + expect(first.sql).toContain('order by tier, shuffle, id'); + expect(first.params).toEqual(expect.arrayContaining(['7', '2026-10-04'])); + }); + + it('shuffles the filtered highlights by a hash of user, seed and id', () => { + const { sql, params } = render(seededReviewSql(scope, db, { filters: { colors: ['blue'] }, seed: -42, limit: 200 })); + expect(sql).toContain('"annotations"."color" in ('); + expect(sql).toContain('order by md5($'); + expect(params).toEqual(expect.arrayContaining(['7', '-42', 'blue', 200])); + }); +}); diff --git a/server/src/modules/notebook/notebook-sql.ts b/server/src/modules/notebook/notebook-sql.ts new file mode 100644 index 000000000..50013d021 --- /dev/null +++ b/server/src/modules/notebook/notebook-sql.ts @@ -0,0 +1,444 @@ +import { and, inArray, or, sql, type SQL, type SQLWrapper } from 'drizzle-orm'; +import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; + +import type { NotebookEntryKind } from '@bookorbit/types'; + +import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils'; +import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils'; +import * as schema from '../../db/schema'; +import { + annotationPositions, + annotations, + authors, + bookAuthors, + bookJournalEntries, + bookMetadata, + bookmarks, + books, + userBookNotes, +} from '../../db/schema'; +import type { NotebookCursor } from './notebook-cursor'; +import { NOTEBOOK_KIND_RANK, hasHighlightOnlyFilter, type NotebookEntryFilters, type NotebookScope } from './notebook-filters'; + +type Db = NodePgDatabase; + +export type NotebookStatus = 'active' | 'trashed'; +export type NotebookDirection = 'asc' | 'desc'; +export type NotebookSortColumn = 'written' | 'edited' | 'deleted'; + +/** One kind's table, described once so every listing, count and deck reads it the same way. */ +export interface NotebookKindSource { + kind: NotebookEntryKind; + rank: number; + table: SQLWrapper; + id: SQLWrapper; + bookId: SQLWrapper; + userId: SQLWrapper; + /** When it was written; a highlight's device may backdate it. */ + writtenAt: SQL; + editedAt: SQL; + deletedAt: SQLWrapper | null; + /** The entry's own text, matched by `q`. */ + ownText: SQLWrapper[]; + /** The note `hasNote` asks about; null for a kind that always counts as having one. */ + note: SQLWrapper | null; + /** Null for a kind with no origin of its own, which counts as `web`. */ + origin: SQLWrapper | null; + /** Conditions a row must always meet to be an entry at all. */ + always: SQL[]; +} + +/** Non-blank after trimming whitespace of any kind, newlines included. */ +export function hasTextSql(value: SQLWrapper): SQL { + return sql`${value} ~ '[^[:space:]]'`; +} + +/** The sort timestamp as text, to the microsecond and independent of the session time zone. */ +export function sortKeySql(value: SQLWrapper): SQL { + return sql`to_char(${value} at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')`; +} + +export const NOTEBOOK_SOURCES: Record = { + highlight: { + kind: 'highlight', + rank: NOTEBOOK_KIND_RANK.highlight, + table: annotations, + id: annotations.id, + bookId: annotations.bookId, + userId: annotations.userId, + // Must stay the exact expression `annotations_user_created_active_idx` is built on. + writtenAt: sql`coalesce(${annotations.sourceCreatedAt}, ${annotations.createdAt})`, + editedAt: sql`${annotations.updatedAt}`, + deletedAt: annotations.deletedAt, + ownText: [annotations.text, annotations.note], + note: annotations.note, + origin: annotations.origin, + always: [], + }, + journal: { + kind: 'journal', + rank: NOTEBOOK_KIND_RANK.journal, + table: bookJournalEntries, + id: bookJournalEntries.id, + bookId: bookJournalEntries.bookId, + userId: bookJournalEntries.userId, + writtenAt: sql`${bookJournalEntries.createdAt}`, + editedAt: sql`${bookJournalEntries.updatedAt}`, + deletedAt: bookJournalEntries.deletedAt, + ownText: [bookJournalEntries.body, bookJournalEntries.quote], + note: null, + origin: null, + always: [], + }, + bookmark: { + kind: 'bookmark', + rank: NOTEBOOK_KIND_RANK.bookmark, + table: bookmarks, + id: bookmarks.id, + bookId: bookmarks.bookId, + userId: bookmarks.userId, + writtenAt: sql`${bookmarks.createdAt}`, + editedAt: sql`${bookmarks.updatedAt}`, + deletedAt: bookmarks.deletedAt, + ownText: [bookmarks.title, bookmarks.note], + note: bookmarks.note, + origin: bookmarks.origin, + always: [], + }, + review: { + kind: 'review', + rank: NOTEBOOK_KIND_RANK.review, + table: userBookNotes, + // A review is one row per (user, book), so its book id is its id. + id: userBookNotes.bookId, + bookId: userBookNotes.bookId, + userId: userBookNotes.userId, + writtenAt: sql`${userBookNotes.updatedAt}`, + editedAt: sql`${userBookNotes.updatedAt}`, + deletedAt: null, + ownText: [userBookNotes.note], + note: null, + origin: null, + always: [hasTextSql(userBookNotes.note)], + }, +}; + +export function sortColumnSql(source: NotebookKindSource, column: NotebookSortColumn): SQL { + if (column === 'written') return source.writtenAt; + if (column === 'edited') return source.editedAt; + if (!source.deletedAt) throw new RangeError(`${source.kind} has no deletion time`); + return sql`${source.deletedAt}`; +} + +function sqlList(values: readonly (string | number)[]): SQL { + return sql.join( + values.map((value) => sql`${value}`), + sql`, `, + ); +} + +/** + * Not anchored exactly: no canonical CFI, an empty one, or one whose status is anything but + * `exact`. This is the rule the iOS notebook draws its badge by from `cfi` and `positionStatus`, + * so the count and the badges agree. A PDF highlight has no CFI row and so counts here too. + */ +export function approximateSql(): SQL { + return sql`not exists ( + select 1 from ${annotationPositions} + where ${annotationPositions.annotationId} = ${annotations.id} + and ${annotationPositions.format} = 'cfi' + and ${annotationPositions.status} = 'exact' + and ${annotationPositions.pos0} is not null + and ${annotationPositions.pos0} <> '' + )`; +} + +/** The book's title or any of its authors, for a book id expression from the outer row. */ +export function bookSearchSql(bookId: SQLWrapper, pattern: string): SQL { + return sql`( + exists (select 1 from ${bookMetadata} where ${bookMetadata.bookId} = ${bookId} and ${accentInsensitiveIlike(bookMetadata.title, pattern)}) + or exists ( + select 1 from ${bookAuthors} inner join ${authors} on ${authors.id} = ${bookAuthors.authorId} + where ${bookAuthors.bookId} = ${bookId} and ${accentInsensitiveIlike(authors.name, pattern)} + ) + )`; +} + +export function entrySearchSql(source: NotebookKindSource, pattern: string): SQL { + const own = source.ownText.map((column) => accentInsensitiveIlike(column, pattern)); + return sql`(${sql.join([...own, bookSearchSql(source.bookId, pattern)], sql` or `)})`; +} + +/** Library access and content rules, evaluated on `books`, which every arm joins under that name. */ +export function scopeSql(scope: NotebookScope, db: Db): SQL[] { + return [inArray(books.libraryId, scope.libraryIds), ...(scope.contentFilters ? buildContentFilterClauses(scope.contentFilters, db) : [])]; +} + +export function statusSql(source: NotebookKindSource, status: NotebookStatus): SQL[] { + if (!source.deletedAt) return status === 'active' ? [] : [sql`false`]; + return [status === 'active' ? sql`${source.deletedAt} is null` : sql`${source.deletedAt} is not null`]; +} + +/** The filter conditions for one kind. A kind a filter rules out gets a `false` rather than vanishing silently. */ +export function filterSql(source: NotebookKindSource, filters: NotebookEntryFilters): SQL[] { + const conditions: SQL[] = []; + if (filters.bookId !== undefined) conditions.push(sql`${source.bookId} = ${filters.bookId}`); + if (filters.origins) { + if (source.origin) conditions.push(sql`${source.origin} in (${sqlList(filters.origins)})`); + else if (!filters.origins.includes('web')) conditions.push(sql`false`); + } + if (filters.hasNote && source.note) conditions.push(hasTextSql(source.note)); + if (source.kind === 'highlight') { + if (filters.starred) conditions.push(sql`${annotations.starredAt} is not null`); + if (filters.colors) conditions.push(sql`${annotations.color} in (${sqlList(filters.colors)})`); + if (filters.approximate) conditions.push(approximateSql()); + } else if (hasHighlightOnlyFilter(filters)) { + conditions.push(sql`false`); + } + if (filters.searchPattern) conditions.push(entrySearchSql(source, filters.searchPattern)); + return conditions; +} + +/** `from inner join books`, the shape every scoped arm reads from. */ +export function scopedFromSql(source: NotebookKindSource): SQL { + return sql`${source.table} inner join ${books} on ${books.id} = ${source.bookId}`; +} + +export function armWhereSql( + source: NotebookKindSource, + scope: NotebookScope, + db: Db, + options: { status: NotebookStatus; filters?: NotebookEntryFilters; extra?: SQL[] }, +): SQL[] { + return [ + sql`${source.userId} = ${scope.userId}`, + ...statusSql(source, options.status), + ...source.always, + ...scopeSql(scope, db), + ...(options.filters ? filterSql(source, options.filters) : []), + ...(options.extra ?? []), + ]; +} + +/** + * Rows after the cursor in the total order `(sortTs, kindRank, id)`, all descending or all + * ascending. Within one arm the kind is fixed, so the tuple comparison collapses to a bound on the + * timestamp for the other kinds and a `(sortTs, id)` row comparison for the cursor's own kind. + */ +export function cursorSql(source: NotebookKindSource, sortTs: SQL, cursor: NotebookCursor, direction: NotebookDirection): SQL { + const key = sql`${cursor.key}::timestamptz`; + const id = sql`${cursor.id}::int`; + if (direction === 'desc') { + if (source.rank < cursor.rank) return sql`${sortTs} <= ${key}`; + if (source.rank > cursor.rank) return sql`${sortTs} < ${key}`; + return sql`(${sortTs}, ${source.id}) < (${key}, ${id})`; + } + if (source.rank < cursor.rank) return sql`${sortTs} > ${key}`; + if (source.rank > cursor.rank) return sql`${sortTs} >= ${key}`; + return sql`(${sortTs}, ${source.id}) > (${key}, ${id})`; +} + +export interface KeysetArm { + source: NotebookKindSource; + where: SQL[]; + sortTs: SQL; +} + +/** + * A UNION ALL of per-kind arms, each ordered on its own index-friendly sort and cut at `limit` + * before the merge, so a deep page reads no more rows than the first one. + */ +export function keysetUnionSql(arms: KeysetArm[], options: { direction: NotebookDirection; limit: number; cursor?: NotebookCursor }): SQL { + const direction = sql.raw(options.direction === 'desc' ? 'desc' : 'asc'); + const parts = arms.map(({ source, where, sortTs }) => { + const conditions = options.cursor ? [...where, cursorSql(source, sortTs, options.cursor, options.direction)] : where; + return sql`( + select ${source.rank}::int as kind_rank, ${source.id} as id, ${source.bookId} as book_id, ${sortTs} as sort_ts + from ${scopedFromSql(source)} + where ${and(...conditions)} + order by ${sortTs} ${direction}, ${source.id} ${direction} + limit ${options.limit} + )`; + }); + return sql` + select kind_rank, id, book_id, ${sortKeySql(sql`sort_ts`)} as sort_key + from (${sql.join(parts, sql` union all `)}) as notebook_entries + order by sort_ts ${direction}, kind_rank ${direction}, id ${direction} + limit ${options.limit} + `; +} + +/** The counts behind the overview, in one statement. A kind left out by the filters counts zero. */ +export function overviewSql(scope: NotebookScope, db: Db, filters: NotebookEntryFilters, kinds: readonly NotebookEntryKind[]): SQL { + const where = (kind: NotebookEntryKind) => { + const source = NOTEBOOK_SOURCES[kind]; + const conditions = armWhereSql(source, scope, db, { status: 'active', filters }); + return and(...(kinds.includes(kind) ? conditions : [...conditions, sql`false`]))!; + }; + const { highlight, journal, bookmark, review } = NOTEBOOK_SOURCES; + // `approximateSql()` as a joined row rather than a NOT EXISTS in the select list: the planner + // prices a select-list subplan per row, which on a large library tips the statement over the JIT + // threshold and costs far more in compilation than the count itself. At most one CFI row exists + // per highlight, so the join cannot multiply the counts. + return sql` + with + h as ( + select ${annotations.bookId} as book_id, ${annotations.color} as color, ${annotations.origin} as origin, + ${annotations.starredAt} is not null as starred, + coalesce(${hasTextSql(annotations.note)}, false) as has_note, + not coalesce(cfi_position.status = 'exact' and cfi_position.pos0 <> '', false) as approximate + from ${scopedFromSql(highlight)} + left join ${annotationPositions} as cfi_position + on cfi_position.annotation_id = ${annotations.id} and cfi_position.format = 'cfi' + where ${where('highlight')} + ), + j as (select ${bookJournalEntries.bookId} as book_id from ${scopedFromSql(journal)} where ${where('journal')}), + b as ( + select ${bookmarks.bookId} as book_id, coalesce(${hasTextSql(bookmarks.note)}, false) as has_note + from ${scopedFromSql(bookmark)} + where ${where('bookmark')} + ), + r as (select ${userBookNotes.bookId} as book_id from ${scopedFromSql(review)} where ${where('review')}) + select + (select count(*) from h)::int as highlights, + (select count(*) from j)::int as journal, + (select count(*) from b)::int as bookmarks, + (select count(*) from r)::int as reviews, + (select count(*) from h where starred)::int as starred, + ((select count(*) from h where has_note) + (select count(*) from b where has_note) + (select count(*) from j) + (select count(*) from r))::int as with_notes, + (select count(*) from h where approximate)::int as approximate, + (select count(*) from (select book_id from h union select book_id from j union select book_id from b union select book_id from r) as counted)::int as books, + ( + select coalesce(json_agg(json_build_object('color', color, 'count', total) order by total desc, color), '[]'::json) + from (select color, count(*)::int as total from h group by color) as by_color + ) as colors, + ( + select coalesce(json_agg(json_build_object('origin', origin, 'count', total) order by total desc, origin), '[]'::json) + from (select origin, count(*)::int as total from h group by origin) as by_origin + ) as origins + `; +} + +/** + * Books with entries, newest activity first. The activity is folded per book from all four kinds + * before the scope join, and the page is cut by the keyset `(lastActivity, bookId)`. + */ +export function bookActivitySql(scope: NotebookScope, db: Db, options: { searchPattern?: string; cursor?: NotebookCursor; limit: number }): SQL { + const arms = Object.values(NOTEBOOK_SOURCES).map( + (source) => sql` + select ${source.bookId} as book_id, ${source.writtenAt} as ts, ${source.rank}::int as kind_rank + from ${source.table} + where ${and(sql`${source.userId} = ${scope.userId}`, ...statusSql(source, 'active'), ...source.always)} + `, + ); + const conditions = [...scopeSql(scope, db)]; + if (options.searchPattern) conditions.push(bookSearchSql(books.id, options.searchPattern)); + if (options.cursor) { + conditions.push(sql`(activity.last_ts, activity.book_id) < (${options.cursor.key}::timestamptz, ${options.cursor.id}::int)`); + } + return sql` + with activity as ( + select book_id, max(ts) as last_ts, + (count(*) filter (where kind_rank = ${NOTEBOOK_KIND_RANK.highlight}))::int as highlights, + (count(*) filter (where kind_rank = ${NOTEBOOK_KIND_RANK.journal}))::int as journal, + (count(*) filter (where kind_rank = ${NOTEBOOK_KIND_RANK.bookmark}))::int as bookmarks, + bool_or(kind_rank = ${NOTEBOOK_KIND_RANK.review}) as has_review + from (${sql.join(arms, sql` union all `)}) as entries + group by book_id + ) + select activity.book_id, activity.highlights, activity.journal, activity.bookmarks, activity.has_review, + ${sortKeySql(sql`activity.last_ts`)} as last_key + from activity inner join ${books} on ${books.id} = activity.book_id + where ${and(...conditions)} + order by activity.last_ts desc, activity.book_id desc + limit ${options.limit} + `; +} + +/** Each listed book's newest entry, one index probe per kind per book. */ +export function latestEntryPerBookSql(userId: number, bookIds: readonly number[]): SQL { + const page = sql.join( + bookIds.map((bookId) => sql`(${bookId}::int)`), + sql`, `, + ); + const arms = Object.values(NOTEBOOK_SOURCES).map( + (source) => sql`( + select ${source.rank}::int as kind_rank, ${source.id} as id, ${source.bookId} as book_id, ${source.writtenAt} as sort_ts + from ${source.table} + where ${and(sql`${source.userId} = ${userId}`, sql`${source.bookId} = page.book_id`, ...statusSql(source, 'active'), ...source.always)} + order by ${source.writtenAt} desc, ${source.id} desc + limit 1 + )`, + ); + return sql` + select latest.kind_rank, latest.id, latest.book_id, ${sortKeySql(sql`latest.sort_ts`)} as sort_key + from (values ${page}) as page(book_id) + cross join lateral ( + select * from (${sql.join(arms, sql` union all `)}) as candidates + order by sort_ts desc, kind_rank desc, id desc + limit 1 + ) as latest + `; +} + +/** + * The earliest write time across every kind, or null with no history. Returned in UTC so the + * reader's zone is applied by the same calendar code that builds the day ranges, never by a + * Postgres zone name that might be missing from the server's tz database. + */ +export function firstEntryAtSql(userId: number): SQL { + const mins = Object.values(NOTEBOOK_SOURCES).map( + (source) => + sql`(select min(${source.writtenAt}) from ${source.table} where ${and(sql`${source.userId} = ${userId}`, ...statusSql(source, 'active'), ...source.always)})`, + ); + return sql`select ${sortKeySql(sql`least(${sql.join(mins, sql`, `)})`)} as first_at`; +} + +/** Half-open `[start, end)` ranges, one per earlier year that can hold the day. */ +export function inRangesSql(value: SQL, ranges: readonly { start: string; end: string }[]): SQL { + if (ranges.length === 0) return sql`false`; + return or(...ranges.map((range) => sql`(${value} >= ${range.start}::timestamptz and ${value} < ${range.end}::timestamptz)`))!; +} + +export const REVIEW_TIER_STARRED = 0; +export const REVIEW_TIER_OLDER = 1; +export const REVIEW_TIER_RECENT = 2; + +/** + * The day's deck: at most one highlight per book, starred ones first, then ones written before + * `olderThan`, then the rest. Inside a tier the order is an md5 of (user, date, id), so the deck + * is the same all day and different the next. + */ +export function dailyReviewSql(scope: NotebookScope, db: Db, options: { date: string; olderThan: string; limit: number }): SQL { + const source = NOTEBOOK_SOURCES.highlight; + const where = armWhereSql(source, scope, db, { status: 'active', extra: [hasTextSql(annotations.text)] }); + const tier = sql`case + when ${annotations.starredAt} is not null then ${REVIEW_TIER_STARRED}::int + when ${source.writtenAt} < ${options.olderThan}::timestamptz then ${REVIEW_TIER_OLDER}::int + else ${REVIEW_TIER_RECENT}::int + end`; + const shuffle = sql`md5(${String(scope.userId)}::text || ':' || ${options.date}::text || ':' || ${annotations.id}::text)`; + return sql` + select id, book_id from ( + select distinct on (${annotations.bookId}) ${annotations.id} as id, ${annotations.bookId} as book_id, ${tier} as tier, ${shuffle} as shuffle + from ${scopedFromSql(source)} + where ${and(...where)} + order by ${annotations.bookId}, tier, shuffle + ) as candidates + order by tier, shuffle, id + limit ${options.limit} + `; +} + +/** A shuffle of the filtered highlights that the same seed repeats. */ +export function seededReviewSql(scope: NotebookScope, db: Db, options: { filters: NotebookEntryFilters; seed: number; limit: number }): SQL { + const source = NOTEBOOK_SOURCES.highlight; + const where = armWhereSql(source, scope, db, { status: 'active', filters: options.filters, extra: [hasTextSql(annotations.text)] }); + return sql` + select ${annotations.id} as id, ${annotations.bookId} as book_id + from ${scopedFromSql(source)} + where ${and(...where)} + order by md5(${String(scope.userId)}::text || ':' || ${String(options.seed)}::text || ':' || ${annotations.id}::text), ${annotations.id} + limit ${options.limit} + `; +} diff --git a/server/src/modules/notebook/notebook.controller.test.ts b/server/src/modules/notebook/notebook.controller.test.ts new file mode 100644 index 000000000..a8904506d --- /dev/null +++ b/server/src/modules/notebook/notebook.controller.test.ts @@ -0,0 +1,74 @@ +import 'reflect-metadata'; + +import { RequestMethod } from '@nestjs/common'; +import { METHOD_METADATA, PATH_METADATA } from '@nestjs/common/constants'; + +import { EMPTY_CONTENT_FILTER_RULES } from '@bookorbit/types'; + +import type { RequestUser } from '../../common/types/request-user'; +import { NotebookController } from './notebook.controller'; + +const user: RequestUser = { + id: 8, + username: 'reader', + name: 'Reader', + email: null, + active: true, + isDefaultPassword: false, + tokenVersion: 1, + settings: {}, + avatarUrl: null, + provisioningMethod: 'local', + isSuperuser: false, + permissions: [], + contentFilters: EMPTY_CONTENT_FILTER_RULES, +}; + +describe('NotebookController', () => { + const service = { + entries: vi.fn(), + overview: vi.fn(), + books: vi.fn(), + review: vi.fn(), + onThisDay: vi.fn(), + trash: vi.fn(), + }; + const controller = new NotebookController(service as never); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('is mounted at notebook', () => { + expect(Reflect.getMetadata(PATH_METADATA, NotebookController)).toBe('notebook'); + }); + + it.each([ + ['entries', 'entries'], + ['overview', 'overview'], + ['books', 'books'], + ['review', 'review'], + ['onThisDay', 'on-this-day'], + ['trash', 'trash'], + ] as const)('declares %s as GET %s', (method, path) => { + const handler = NotebookController.prototype[method]; + expect(Reflect.getMetadata(METHOD_METADATA, handler)).toBe(RequestMethod.GET); + expect(Reflect.getMetadata(PATH_METADATA, handler)).toBe(path); + }); + + it('delegates every route with the current user and the query', async () => { + await controller.entries(user, { sort: 'oldest' }); + await controller.overview(user, { starred: true }); + await controller.books(user, { q: 'dune' }); + await controller.review(user, { seed: 1 }); + await controller.onThisDay(user, { date: '2026-10-04' }); + await controller.trash(user, { limit: 5 }); + + expect(service.entries).toHaveBeenCalledWith(user, { sort: 'oldest' }); + expect(service.overview).toHaveBeenCalledWith(user, { starred: true }); + expect(service.books).toHaveBeenCalledWith(user, { q: 'dune' }); + expect(service.review).toHaveBeenCalledWith(user, { seed: 1 }); + expect(service.onThisDay).toHaveBeenCalledWith(user, { date: '2026-10-04' }); + expect(service.trash).toHaveBeenCalledWith(user, { limit: 5 }); + }); +}); diff --git a/server/src/modules/notebook/notebook.controller.ts b/server/src/modules/notebook/notebook.controller.ts new file mode 100644 index 000000000..f2309147e --- /dev/null +++ b/server/src/modules/notebook/notebook.controller.ts @@ -0,0 +1,58 @@ +import { Controller, Get, Query } from '@nestjs/common'; + +import type { + NotebookBooksResponse, + NotebookEntriesResponse, + NotebookOnThisDayResponse, + NotebookOverview, + NotebookReviewResponse, + NotebookTrashResponse, +} from '@bookorbit/types'; + +import { CurrentUser } from '../../common/decorators/current-user.decorator'; +import type { RequestUser } from '../../common/types/request-user'; +import { + NotebookBooksQueryDto, + NotebookEntriesQueryDto, + NotebookOnThisDayQueryDto, + NotebookOverviewQueryDto, + NotebookReviewQueryDto, + NotebookTrashQueryDto, +} from './dto/notebook-query.dto'; +import { NotebookService } from './notebook.service'; + +/** The library-wide Notebook. Read only: edits go through each kind's own per-book routes. */ +@Controller('notebook') +export class NotebookController { + constructor(private readonly notebookService: NotebookService) {} + + @Get('entries') + entries(@CurrentUser() user: RequestUser, @Query() query: NotebookEntriesQueryDto): Promise { + return this.notebookService.entries(user, query); + } + + @Get('overview') + overview(@CurrentUser() user: RequestUser, @Query() query: NotebookOverviewQueryDto): Promise { + return this.notebookService.overview(user, query); + } + + @Get('books') + books(@CurrentUser() user: RequestUser, @Query() query: NotebookBooksQueryDto): Promise { + return this.notebookService.books(user, query); + } + + @Get('review') + review(@CurrentUser() user: RequestUser, @Query() query: NotebookReviewQueryDto): Promise { + return this.notebookService.review(user, query); + } + + @Get('on-this-day') + onThisDay(@CurrentUser() user: RequestUser, @Query() query: NotebookOnThisDayQueryDto): Promise { + return this.notebookService.onThisDay(user, query); + } + + @Get('trash') + trash(@CurrentUser() user: RequestUser, @Query() query: NotebookTrashQueryDto): Promise { + return this.notebookService.trash(user, query); + } +} diff --git a/server/src/modules/notebook/notebook.module.test.ts b/server/src/modules/notebook/notebook.module.test.ts new file mode 100644 index 000000000..f18db0caf --- /dev/null +++ b/server/src/modules/notebook/notebook.module.test.ts @@ -0,0 +1,18 @@ +import 'reflect-metadata'; + +import { MODULE_METADATA } from '@nestjs/common/constants'; + +import { BookCoverStoreModule } from '../book-cover-store/book-cover-store.module'; +import { LibraryModule } from '../library/library.module'; +import { NotebookController } from './notebook.controller'; +import { NotebookModule } from './notebook.module'; +import { NotebookRepository } from './notebook.repository'; +import { NotebookService } from './notebook.service'; + +describe('NotebookModule', () => { + it('registers its controller and providers and imports library access and covers', () => { + expect(Reflect.getMetadata(MODULE_METADATA.IMPORTS, NotebookModule)).toEqual(expect.arrayContaining([LibraryModule, BookCoverStoreModule])); + expect(Reflect.getMetadata(MODULE_METADATA.CONTROLLERS, NotebookModule)).toEqual([NotebookController]); + expect(Reflect.getMetadata(MODULE_METADATA.PROVIDERS, NotebookModule)).toEqual(expect.arrayContaining([NotebookService, NotebookRepository])); + }); +}); diff --git a/server/src/modules/notebook/notebook.module.ts b/server/src/modules/notebook/notebook.module.ts new file mode 100644 index 000000000..dec29c53b --- /dev/null +++ b/server/src/modules/notebook/notebook.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; + +import { BookCoverStoreModule } from '../book-cover-store/book-cover-store.module'; +import { LibraryModule } from '../library/library.module'; +import { NotebookController } from './notebook.controller'; +import { NotebookRepository } from './notebook.repository'; +import { NotebookService } from './notebook.service'; + +@Module({ + imports: [LibraryModule, BookCoverStoreModule], + controllers: [NotebookController], + providers: [NotebookService, NotebookRepository], +}) +export class NotebookModule {} diff --git a/server/src/modules/notebook/notebook.repository.ts b/server/src/modules/notebook/notebook.repository.ts new file mode 100644 index 000000000..1724514c8 --- /dev/null +++ b/server/src/modules/notebook/notebook.repository.ts @@ -0,0 +1,354 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { and, asc, desc, eq, inArray, isNull, sql, type SQL } from 'drizzle-orm'; +import { NodePgDatabase } from 'drizzle-orm/node-postgres'; + +import { AUDIO_FORMAT_LIST, BOOK_FORMATS, isAudioFormat, type NotebookEntryKind } from '@bookorbit/types'; + +import { DB } from '../../db'; +import * as schema from '../../db/schema'; +import { + annotationPositions, + annotations, + authors, + bookAuthors, + bookFiles, + bookJournalEntries, + bookMetadata, + bookmarks, + books, + libraries, + userBookNotes, + userBookRatings, +} from '../../db/schema'; +import type { NotebookCursor } from './notebook-cursor'; +import type { NotebookEntryFilters, NotebookScope } from './notebook-filters'; +import type { NotebookBookmarkRow, NotebookHighlightRow, NotebookJournalRow, NotebookLabelRow, NotebookReviewRow } from './notebook-mapper'; +import { + NOTEBOOK_SOURCES, + armWhereSql, + bookActivitySql, + dailyReviewSql, + firstEntryAtSql, + inRangesSql, + keysetUnionSql, + latestEntryPerBookSql, + overviewSql, + seededReviewSql, + sortColumnSql, + type NotebookDirection, + type NotebookSortColumn, + type NotebookStatus, +} from './notebook-sql'; + +type Db = NodePgDatabase; + +/** Where an entry sits in a listing, before its row is loaded. */ +export interface NotebookEntryKey { + kindRank: number; + id: number; + bookId: number; + sortKey: string; +} + +export interface NotebookOverviewRow { + highlights: number; + journal: number; + bookmarks: number; + reviews: number; + starred: number; + withNotes: number; + approximate: number; + books: number; + colors: { color: string; count: number }[]; + origins: { origin: string; count: number }[]; +} + +export interface NotebookBookActivityRow { + bookId: number; + highlights: number; + journal: number; + bookmarks: number; + hasReview: boolean; + lastKey: string; +} + +type RawKeyRow = { kind_rank: number; id: number; book_id: number; sort_key: string }; + +const READABLE_FORMATS = BOOK_FORMATS.filter((format) => !isAudioFormat(format)); + +function sqlList(values: readonly string[]): SQL { + return sql.join( + values.map((value) => sql`${value}`), + sql`, `, + ); +} + +/** + * The file Open in Book opens: the book's primary file when it is readable, otherwise its best + * readable content file by the default format priority. Audio never qualifies. + */ +function readFileSql(pick: SQL): SQL { + return sql`( + select ${pick} from ${bookFiles} + where ${bookFiles.bookId} = ${books.id} + and ${bookFiles.role} in ('content', 'primary') + and lower(${bookFiles.format}) in (${sqlList(READABLE_FORMATS)}) + order by coalesce(${bookFiles.id} = ${books.primaryFileId}, false) desc, + array_position(array[${sqlList(READABLE_FORMATS)}]::text[], lower(${bookFiles.format})), + ${bookFiles.sortOrder} asc nulls last, + ${bookFiles.id} asc + limit 1 + )`; +} + +function toKey(row: RawKeyRow): NotebookEntryKey { + return { + kindRank: Number(row.kind_rank), + id: Number(row.id), + bookId: Number(row.book_id), + sortKey: row.sort_key, + }; +} + +@Injectable() +export class NotebookRepository { + constructor(@Inject(DB) private readonly db: Db) {} + + async findEntryKeys( + scope: NotebookScope, + params: { + kinds: readonly NotebookEntryKind[]; + status: NotebookStatus; + filters?: NotebookEntryFilters; + sortColumn: NotebookSortColumn; + direction: NotebookDirection; + cursor?: NotebookCursor; + limit: number; + }, + ): Promise { + if (params.kinds.length === 0 || scope.libraryIds.length === 0) return []; + const arms = params.kinds.map((kind) => { + const source = NOTEBOOK_SOURCES[kind]; + return { + source, + sortTs: sortColumnSql(source, params.sortColumn), + where: armWhereSql(source, scope, this.db, { status: params.status, filters: params.filters }), + }; + }); + const result = await this.db.execute( + keysetUnionSql(arms, { direction: params.direction, limit: params.limit, cursor: params.cursor }), + ); + return result.rows.map(toKey); + } + + async findOnThisDayKeys(scope: NotebookScope, params: { ranges: { start: string; end: string }[]; limit: number }): Promise { + if (params.ranges.length === 0 || scope.libraryIds.length === 0) return []; + const arms = Object.values(NOTEBOOK_SOURCES).map((source) => ({ + source, + sortTs: source.writtenAt, + where: armWhereSql(source, scope, this.db, { status: 'active', extra: [inRangesSql(source.writtenAt, params.ranges)] }), + })); + const result = await this.db.execute(keysetUnionSql(arms, { direction: 'desc', limit: params.limit })); + return result.rows.map(toKey); + } + + /** The reader's oldest active entry of any kind, as a sort key, or null with no history. */ + async findFirstEntryAt(userId: number): Promise { + const result = await this.db.execute<{ first_at: string | null }>(firstEntryAtSql(userId)); + return result.rows[0]?.first_at ?? null; + } + + async findOverview(scope: NotebookScope, filters: NotebookEntryFilters, kinds: readonly NotebookEntryKind[]): Promise { + const result = await this.db.execute<{ + highlights: number; + journal: number; + bookmarks: number; + reviews: number; + starred: number; + with_notes: number; + approximate: number; + books: number; + colors: { color: string; count: number }[] | null; + origins: { origin: string; count: number }[] | null; + }>(overviewSql(scope, this.db, filters, kinds)); + const row = result.rows[0]; + return { + highlights: Number(row?.highlights ?? 0), + journal: Number(row?.journal ?? 0), + bookmarks: Number(row?.bookmarks ?? 0), + reviews: Number(row?.reviews ?? 0), + starred: Number(row?.starred ?? 0), + withNotes: Number(row?.with_notes ?? 0), + approximate: Number(row?.approximate ?? 0), + books: Number(row?.books ?? 0), + colors: (row?.colors ?? []).map((entry) => ({ color: entry.color, count: Number(entry.count) })), + origins: (row?.origins ?? []).map((entry) => ({ origin: entry.origin, count: Number(entry.count) })), + }; + } + + async findBookActivity( + scope: NotebookScope, + params: { searchPattern?: string; cursor?: NotebookCursor; limit: number }, + ): Promise { + if (scope.libraryIds.length === 0) return []; + const result = await this.db.execute<{ + book_id: number; + highlights: number; + journal: number; + bookmarks: number; + has_review: boolean | null; + last_key: string; + }>(bookActivitySql(scope, this.db, params)); + return result.rows.map((row) => ({ + bookId: Number(row.book_id), + highlights: Number(row.highlights), + journal: Number(row.journal), + bookmarks: Number(row.bookmarks), + hasReview: row.has_review === true, + lastKey: row.last_key, + })); + } + + async findLatestEntryPerBook(userId: number, bookIds: readonly number[]): Promise { + if (bookIds.length === 0) return []; + const result = await this.db.execute(latestEntryPerBookSql(userId, bookIds)); + return result.rows.map(toKey); + } + + /** Every active highlight colour per book with its count, heaviest first. */ + async findColorMix(userId: number, bookIds: readonly number[]): Promise<{ bookId: number; color: string; count: number }[]> { + if (bookIds.length === 0) return []; + const total = sql`count(*)::int`; + return this.db + .select({ bookId: annotations.bookId, color: annotations.color, count: total }) + .from(annotations) + .where(and(eq(annotations.userId, userId), isNull(annotations.deletedAt), inArray(annotations.bookId, [...bookIds]))) + .groupBy(annotations.bookId, annotations.color) + .orderBy(asc(annotations.bookId), desc(total), asc(annotations.color)); + } + + async findDailyReview(scope: NotebookScope, params: { date: string; olderThan: string; limit: number }): Promise<{ id: number; bookId: number }[]> { + if (scope.libraryIds.length === 0) return []; + const result = await this.db.execute<{ id: number; book_id: number }>(dailyReviewSql(scope, this.db, params)); + return result.rows.map((row) => ({ id: Number(row.id), bookId: Number(row.book_id) })); + } + + async findSeededReview( + scope: NotebookScope, + params: { filters: NotebookEntryFilters; seed: number; limit: number }, + ): Promise<{ id: number; bookId: number }[]> { + if (scope.libraryIds.length === 0) return []; + const result = await this.db.execute<{ id: number; book_id: number }>(seededReviewSql(scope, this.db, params)); + return result.rows.map((row) => ({ id: Number(row.id), bookId: Number(row.book_id) })); + } + + async findHighlights(userId: number, ids: readonly number[]): Promise { + if (ids.length === 0) return []; + return this.db + .select({ + id: annotations.id, + bookId: annotations.bookId, + text: annotations.text, + color: annotations.color, + style: annotations.style, + note: annotations.note, + chapterTitle: annotations.chapterTitle, + origin: annotations.origin, + sourceCreatedAt: annotations.sourceCreatedAt, + createdAt: annotations.createdAt, + updatedAt: annotations.updatedAt, + deletedAt: annotations.deletedAt, + starredAt: annotations.starredAt, + cfi: annotationPositions.pos0, + cfiStatus: annotationPositions.status, + }) + .from(annotations) + .leftJoin(annotationPositions, and(eq(annotationPositions.annotationId, annotations.id), eq(annotationPositions.format, 'cfi'))) + .where(and(eq(annotations.userId, userId), inArray(annotations.id, [...ids]))); + } + + async findJournal(userId: number, ids: readonly number[]): Promise { + if (ids.length === 0) return []; + return this.db + .select({ + id: bookJournalEntries.id, + clientId: bookJournalEntries.clientId, + bookId: bookJournalEntries.bookId, + body: bookJournalEntries.body, + quote: bookJournalEntries.quote, + chapterTitle: bookJournalEntries.chapterTitle, + positionPercent: bookJournalEntries.positionPercent, + cfi: bookJournalEntries.cfi, + positionSeconds: bookJournalEntries.positionSeconds, + createdAt: bookJournalEntries.createdAt, + updatedAt: bookJournalEntries.updatedAt, + deletedAt: bookJournalEntries.deletedAt, + }) + .from(bookJournalEntries) + .where(and(eq(bookJournalEntries.userId, userId), inArray(bookJournalEntries.id, [...ids]))); + } + + async findBookmarks(userId: number, ids: readonly number[]): Promise { + if (ids.length === 0) return []; + return this.db + .select({ + id: bookmarks.id, + clientId: bookmarks.clientId, + bookId: bookmarks.bookId, + title: bookmarks.title, + note: bookmarks.note, + cfi: bookmarks.cfi, + positionSeconds: bookmarks.positionSeconds, + origin: bookmarks.origin, + createdAt: bookmarks.createdAt, + updatedAt: bookmarks.updatedAt, + deletedAt: bookmarks.deletedAt, + }) + .from(bookmarks) + .where(and(eq(bookmarks.userId, userId), inArray(bookmarks.id, [...ids]))); + } + + async findReviews(userId: number, bookIds: readonly number[]): Promise { + if (bookIds.length === 0) return []; + return this.db + .select({ + bookId: userBookNotes.bookId, + note: userBookNotes.note, + updatedAt: userBookNotes.updatedAt, + rating: userBookRatings.rating, + }) + .from(userBookNotes) + .leftJoin(userBookRatings, and(eq(userBookRatings.userId, userBookNotes.userId), eq(userBookRatings.bookId, userBookNotes.bookId))) + .where(and(eq(userBookNotes.userId, userId), inArray(userBookNotes.bookId, [...bookIds]))); + } + + /** One row per book for the label sidecar; cover slots are loaded separately by the cover store. */ + async findLabels(bookIds: readonly number[]): Promise { + if (bookIds.length === 0) return []; + return this.db + .select({ + id: books.id, + title: bookMetadata.title, + author: sql`( + select string_agg(${authors.name}, ', ' order by ${bookAuthors.displayOrder}, ${authors.id}) + from ${bookAuthors} inner join ${authors} on ${authors.id} = ${bookAuthors.authorId} + where ${bookAuthors.bookId} = ${books.id} + )`, + coverSource: bookMetadata.coverSource, + coverAspectRatio: libraries.coverAspectRatio, + updatedAt: books.updatedAt, + readFileId: readFileSql(sql`${bookFiles.id}`), + readFileFormat: readFileSql(sql`lower(${bookFiles.format})`), + hasAudio: sql`exists ( + select 1 from ${bookFiles} + where ${bookFiles.bookId} = ${books.id} + and ${bookFiles.role} in ('content', 'primary') + and lower(${bookFiles.format}) in (${sqlList(AUDIO_FORMAT_LIST)}) + )`, + }) + .from(books) + .innerJoin(libraries, eq(libraries.id, books.libraryId)) + .leftJoin(bookMetadata, eq(bookMetadata.bookId, books.id)) + .where(inArray(books.id, [...bookIds])); + } +} diff --git a/server/src/modules/notebook/notebook.service.test.ts b/server/src/modules/notebook/notebook.service.test.ts new file mode 100644 index 000000000..944bf45e0 --- /dev/null +++ b/server/src/modules/notebook/notebook.service.test.ts @@ -0,0 +1,387 @@ +import { BadRequestException } from '@nestjs/common'; + +import { + EMPTY_CONTENT_FILTER_RULES, + NOTEBOOK_BOOK_COLOR_MIX_MAX, + NOTEBOOK_ON_THIS_DAY_MAX, + NOTEBOOK_REVIEW_DAILY_SIZE, + NOTEBOOK_REVIEW_SHUFFLE_MAX, +} from '@bookorbit/types'; + +import type { RequestUser } from '../../common/types/request-user'; +import { decodeNotebookCursor, encodeNotebookCursor } from './notebook-cursor'; +import { NotebookService } from './notebook.service'; + +const user: RequestUser = { + id: 8, + username: 'reader', + name: 'Reader', + email: null, + active: true, + isDefaultPassword: false, + tokenVersion: 1, + settings: { timezone: 'Asia/Kolkata' }, + avatarUrl: null, + provisioningMethod: 'local', + isSuperuser: false, + permissions: [], + contentFilters: { ...EMPTY_CONTENT_FILTER_RULES, excludeTagIds: [3] }, +}; + +const at = new Date('2026-02-03T04:05:06.000Z'); + +function highlightRow(id: number, bookId: number, overrides: Record = {}) { + return { + id, + bookId, + text: `text ${id}`, + color: 'yellow', + style: 'highlight', + note: null, + chapterTitle: null, + origin: 'web', + sourceCreatedAt: null, + createdAt: at, + updatedAt: at, + deletedAt: null, + starredAt: null, + cfi: null, + cfiStatus: null, + ...overrides, + }; +} + +function labelRow(id: number) { + return { + id, + title: `Book ${id}`, + author: 'Author', + coverSource: 'extracted', + coverAspectRatio: '2/3', + updatedAt: at, + readFileId: id * 10, + readFileFormat: 'epub', + hasAudio: false, + }; +} + +function key(kindRank: number, id: number, bookId: number, sortKey = '2026-02-03T04:05:06.123456Z') { + return { kindRank, id, bookId, sortKey }; +} + +function makeService(libraryIds = [1, 2]) { + const repo = { + findEntryKeys: vi.fn().mockResolvedValue([]), + findOverview: vi.fn(), + findBookActivity: vi.fn().mockResolvedValue([]), + findLatestEntryPerBook: vi.fn().mockResolvedValue([]), + findColorMix: vi.fn().mockResolvedValue([]), + findDailyReview: vi.fn().mockResolvedValue([]), + findSeededReview: vi.fn().mockResolvedValue([]), + findFirstEntryAt: vi.fn().mockResolvedValue(null), + findOnThisDayKeys: vi.fn().mockResolvedValue([]), + findHighlights: vi.fn().mockResolvedValue([]), + findJournal: vi.fn().mockResolvedValue([]), + findBookmarks: vi.fn().mockResolvedValue([]), + findReviews: vi.fn().mockResolvedValue([]), + findLabels: vi.fn().mockImplementation((ids: number[]) => Promise.resolve(ids.map(labelRow))), + }; + const libraryService = { findAccessibleLibraryIds: vi.fn().mockResolvedValue(libraryIds) }; + const coverStore = { + slotsFor: vi.fn().mockResolvedValue(new Map()), + coverVersion: vi.fn((ratio: string, _slots: unknown[], legacy: string) => `legacy:${ratio}:${legacy}`), + }; + const service = new NotebookService(repo as never, libraryService as never, coverStore as never); + return { service, repo, libraryService, coverStore }; +} + +describe('NotebookService.entries', () => { + it('returns an empty page without querying when the reader can open no library', async () => { + const { service, repo } = makeService([]); + await expect(service.entries(user, {})).resolves.toEqual({ items: [], books: {}, nextCursor: null }); + expect(repo.findEntryKeys).not.toHaveBeenCalled(); + expect(repo.findLabels).not.toHaveBeenCalled(); + }); + + it('pages newest first by default with the scope, filters and one extra row', async () => { + const { service, repo } = makeService(); + await service.entries(user, { hasNote: false, starred: true, q: ' 50% ' }); + expect(repo.findEntryKeys).toHaveBeenCalledWith( + { userId: 8, libraryIds: [1, 2], contentFilters: user.contentFilters }, + expect.objectContaining({ + kinds: ['highlight'], + status: 'active', + filters: { starred: true, searchPattern: '%50\\%%' }, + sortColumn: 'written', + direction: 'desc', + cursor: undefined, + limit: 41, + }), + ); + }); + + it('drops content rules for a superuser', async () => { + const { service, repo } = makeService(); + await service.entries({ ...user, isSuperuser: true }, {}); + expect(repo.findEntryKeys.mock.calls[0][0]).toEqual({ userId: 8, libraryIds: [1, 2], contentFilters: undefined }); + }); + + it('skips the query when the filters leave no kind', async () => { + const { service, repo } = makeService(); + await expect(service.entries(user, { kinds: ['journal'], colors: 'blue' })).resolves.toEqual({ items: [], books: {}, nextCursor: null }); + expect(repo.findEntryKeys).not.toHaveBeenCalled(); + }); + + it('maps oldest and edited to their columns and directions', async () => { + const { service, repo } = makeService(); + await service.entries(user, { sort: 'oldest' }); + await service.entries(user, { sort: 'edited' }); + expect(repo.findEntryKeys.mock.calls[0][1]).toMatchObject({ sortColumn: 'written', direction: 'asc' }); + expect(repo.findEntryKeys.mock.calls[1][1]).toMatchObject({ sortColumn: 'edited', direction: 'desc' }); + }); + + it('keeps the key order across kinds, returns labels and a cursor at the last item', async () => { + const { service, repo } = makeService(); + repo.findEntryKeys.mockResolvedValue([key(3, 5, 5, '2026-02-03T04:05:06.999999Z'), key(0, 11, 6), key(1, 2, 6, '2026-01-01T00:00:00.000001Z')]); + repo.findHighlights.mockResolvedValue([highlightRow(11, 6)]); + repo.findReviews.mockResolvedValue([{ bookId: 5, note: 'Great', updatedAt: at, rating: null }]); + const result = await service.entries(user, { limit: 2 }); + expect(result.items.map((item) => [item.kind, item.id])).toEqual([ + ['review', 5], + ['highlight', 11], + ]); + expect(repo.findJournal).toHaveBeenCalledWith(8, []); + expect(Object.keys(result.books).sort()).toEqual(['5', '6']); + expect(result.books['6']).toMatchObject({ id: 6, title: 'Book 6', coverVersion: `legacy:2/3:${at.toISOString()}`, readFileId: 60 }); + expect(decodeNotebookCursor(result.nextCursor!, 'newest')).toEqual({ scope: 'newest', key: '2026-02-03T04:05:06.123456Z', rank: 0, id: 11 }); + }); + + it('has no next cursor on the last page', async () => { + const { service, repo } = makeService(); + repo.findEntryKeys.mockResolvedValue([key(0, 11, 6)]); + repo.findHighlights.mockResolvedValue([highlightRow(11, 6)]); + expect((await service.entries(user, { limit: 2 })).nextCursor).toBeNull(); + }); + + it('passes a valid cursor through and rejects one from another sort', async () => { + const { service, repo } = makeService(); + const cursor = encodeNotebookCursor({ scope: 'oldest', key: '2026-02-03T04:05:06.123456Z', rank: 2, id: 9 }); + await service.entries(user, { sort: 'oldest', cursor }); + expect(repo.findEntryKeys.mock.calls[0][1].cursor).toEqual({ scope: 'oldest', key: '2026-02-03T04:05:06.123456Z', rank: 2, id: 9 }); + await expect(service.entries(user, { sort: 'newest', cursor })).rejects.toBeInstanceOf(BadRequestException); + await expect(service.entries(user, { cursor: 'garbage!' })).rejects.toBeInstanceOf(BadRequestException); + }); + + it('drops a row that was trashed between the keys and the load', async () => { + const { service, repo } = makeService(); + repo.findEntryKeys.mockResolvedValue([key(0, 11, 6), key(0, 12, 6)]); + repo.findHighlights.mockResolvedValue([highlightRow(11, 6), highlightRow(12, 6, { deletedAt: at })]); + expect((await service.entries(user, {})).items.map((item) => item.id)).toEqual([11]); + }); +}); + +describe('NotebookService.overview', () => { + it('is all zeros with no accessible library', async () => { + const { service, repo } = makeService([]); + const overview = await service.overview(user, {}); + expect(overview).toMatchObject({ total: 0, books: 0, colors: [], origins: [] }); + expect(repo.findOverview).not.toHaveBeenCalled(); + }); + + it('sums the kinds into the total and passes the kinds the filters allow', async () => { + const { service, repo } = makeService(); + repo.findOverview.mockResolvedValue({ + highlights: 10, + journal: 2, + bookmarks: 3, + reviews: 1, + starred: 4, + withNotes: 6, + approximate: 5, + books: 7, + colors: [{ color: 'yellow', count: 10 }], + origins: [ + { origin: 'kobo', count: 6 }, + { origin: 'web', count: 4 }, + ], + }); + const overview = await service.overview(user, { origins: ['kobo'] }); + expect(repo.findOverview).toHaveBeenCalledWith(expect.anything(), { origins: ['kobo'] }, ['highlight', 'bookmark']); + expect(overview).toEqual({ + total: 16, + highlights: 10, + journal: 2, + bookmarks: 3, + reviews: 1, + starred: 4, + withNotes: 6, + approximate: 5, + books: 7, + colors: [{ color: 'yellow', count: 10 }], + origins: [ + { origin: 'kobo', count: 6 }, + { origin: 'web', count: 4 }, + ], + }); + }); +}); + +describe('NotebookService.books', () => { + it('builds rows with counts, a capped colour mix, the latest entry and a cursor', async () => { + const { service, repo } = makeService(); + repo.findBookActivity.mockResolvedValue([ + { bookId: 6, highlights: 12, journal: 1, bookmarks: 0, hasReview: true, lastKey: '2026-02-03T04:05:06.123456Z' }, + { bookId: 7, highlights: 0, journal: 0, bookmarks: 1, hasReview: false, lastKey: '2026-02-01T00:00:00.000000Z' }, + ]); + repo.findColorMix.mockResolvedValue(['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((color, index) => ({ bookId: 6, color, count: 10 - index }))); + repo.findLatestEntryPerBook.mockResolvedValue([key(0, 11, 6)]); + repo.findHighlights.mockResolvedValue([highlightRow(11, 6)]); + + const result = await service.books(user, { q: ' herbert ', limit: 1 }); + expect(repo.findBookActivity).toHaveBeenCalledWith(expect.anything(), { searchPattern: '%herbert%', cursor: undefined, limit: 2 }); + expect(repo.findLatestEntryPerBook).toHaveBeenCalledWith(8, [6]); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ + book: { id: 6, title: 'Book 6' }, + highlights: 12, + journal: 1, + bookmarks: 0, + hasReview: true, + lastActivityAt: '2026-02-03T04:05:06.123Z', + latest: { kind: 'highlight', id: 11 }, + }); + expect(result.items[0].colors).toHaveLength(NOTEBOOK_BOOK_COLOR_MIX_MAX); + expect(result.items[0].colors[0]).toEqual({ color: 'a', count: 10 }); + expect(decodeNotebookCursor(result.nextCursor!, 'books')).toMatchObject({ key: '2026-02-03T04:05:06.123456Z', id: 6 }); + }); + + it('is empty with no accessible library', async () => { + const { service, repo } = makeService([]); + await expect(service.books(user, {})).resolves.toEqual({ items: [], nextCursor: null }); + expect(repo.findBookActivity).not.toHaveBeenCalled(); + }); +}); + +describe('NotebookService.review', () => { + it.each([[{}], [{ date: '2026-10-04', seed: 3 }]])('rejects %j: exactly one of date and seed', async (query) => { + const { service } = makeService(); + await expect(service.review(user, query)).rejects.toBeInstanceOf(BadRequestException); + }); + + it('builds the daily deck for the date, with its older-than boundary', async () => { + const { service, repo } = makeService(); + repo.findDailyReview.mockResolvedValue([ + { id: 11, bookId: 6 }, + { id: 4, bookId: 2 }, + ]); + repo.findHighlights.mockResolvedValue([highlightRow(4, 2), highlightRow(11, 6)]); + const deck = await service.review(user, { date: '2026-10-04', starred: true }); + expect(repo.findDailyReview).toHaveBeenCalledWith(expect.anything(), { + date: '2026-10-04', + olderThan: '2026-09-04T00:00:00.000Z', + limit: NOTEBOOK_REVIEW_DAILY_SIZE, + }); + expect(deck.items.map((item) => item.id)).toEqual([11, 4]); + expect(Object.keys(deck.books).sort()).toEqual(['2', '6']); + }); + + it('shuffles by seed with the entry filters', async () => { + const { service, repo } = makeService(); + await service.review(user, { seed: 42, colors: 'blue', hasNote: false }); + expect(repo.findSeededReview).toHaveBeenCalledWith(expect.anything(), { + filters: { colors: ['blue'] }, + seed: 42, + limit: NOTEBOOK_REVIEW_SHUFFLE_MAX, + }); + }); + + it('is empty with no accessible library', async () => { + const { service, repo } = makeService([]); + await expect(service.review(user, { date: '2026-10-04' })).resolves.toEqual({ items: [], books: {} }); + expect(repo.findDailyReview).not.toHaveBeenCalled(); + }); +}); + +describe('NotebookService.onThisDay', () => { + it("defaults to the reader's zone and groups entries by year, newest first", async () => { + const { service, repo } = makeService(); + // Late on 31 December 2022 in UTC is already 2023 in Kolkata. + repo.findFirstEntryAt.mockResolvedValue('2022-12-31T20:00:00.000000Z'); + repo.findOnThisDayKeys.mockResolvedValue([ + key(0, 11, 6, '2025-10-04T18:29:59.999999Z'), + key(1, 3, 6, '2025-10-03T18:30:00.000000Z'), + key(0, 12, 7, '2023-10-04T00:00:00.000000Z'), + key(0, 13, 7, '2025-10-05T00:00:00.000000Z'), + ]); + repo.findHighlights.mockResolvedValue([highlightRow(11, 6), highlightRow(12, 7), highlightRow(13, 7)]); + repo.findJournal.mockResolvedValue([ + { + id: 3, + clientId: '4f9c7a52-3d1e-4b6a-9a51-0c2f6e8d7b13', + bookId: 6, + body: 'b', + quote: null, + chapterTitle: null, + positionPercent: null, + cfi: null, + positionSeconds: null, + createdAt: at, + updatedAt: at, + deletedAt: null, + }, + ]); + + const result = await service.onThisDay(user, { date: '2026-10-04' }); + expect(repo.findFirstEntryAt).toHaveBeenCalledWith(8); + const call = repo.findOnThisDayKeys.mock.calls[0][1]; + expect(call.limit).toBe(NOTEBOOK_ON_THIS_DAY_MAX); + expect(call.ranges.map((range: { year: number }) => range.year)).toEqual([2025, 2024, 2023]); + expect(call.ranges[0]).toEqual({ year: 2025, start: '2025-10-03T18:30:00.000Z', end: '2025-10-04T18:30:00.000Z' }); + expect(result.date).toBe('2026-10-04'); + expect(result.years.map((year) => [year.year, year.items.map((item) => `${item.kind}:${item.id}`)])).toEqual([ + [2025, ['highlight:11', 'journal:3']], + [2023, ['highlight:12']], + ]); + expect(Object.keys(result.books).sort()).toEqual(['6', '7']); + }); + + it('uses the zone the request names and falls back to UTC without a setting', async () => { + const { service, repo } = makeService(); + repo.findFirstEntryAt.mockResolvedValue('2025-01-01T00:00:00.000000Z'); + await service.onThisDay(user, { date: '2026-10-04', tz: 'Europe/Berlin' }); + await service.onThisDay({ ...user, settings: {} }, { date: '2026-10-04' }); + const [berlin, utc] = repo.findOnThisDayKeys.mock.calls.map((call) => call[1].ranges[0]); + expect(berlin).toEqual({ year: 2025, start: '2025-10-03T22:00:00.000Z', end: '2025-10-04T22:00:00.000Z' }); + expect(utc).toEqual({ year: 2025, start: '2025-10-04T00:00:00.000Z', end: '2025-10-05T00:00:00.000Z' }); + }); + + it('is empty with no accessible library', async () => { + const { service, repo } = makeService([]); + await expect(service.onThisDay(user, { date: '2026-10-04' })).resolves.toEqual({ date: '2026-10-04', years: [], books: {} }); + expect(repo.findFirstEntryAt).not.toHaveBeenCalled(); + }); +}); + +describe('NotebookService.trash', () => { + it('lists deleted highlights and journal entries by deletion time with deletedAt set', async () => { + const { service, repo } = makeService(); + repo.findEntryKeys.mockResolvedValue([key(0, 11, 6), key(0, 12, 6)]); + repo.findHighlights.mockResolvedValue([highlightRow(11, 6, { deletedAt: at }), highlightRow(12, 6)]); + const result = await service.trash(user, { limit: 1 }); + expect(repo.findEntryKeys.mock.calls[0][1]).toMatchObject({ + kinds: ['highlight', 'journal'], + status: 'trashed', + sortColumn: 'deleted', + direction: 'desc', + limit: 2, + }); + expect(result.items).toEqual([expect.objectContaining({ id: 11, deletedAt: at.toISOString() })]); + expect(decodeNotebookCursor(result.nextCursor!, 'trash')).toMatchObject({ id: 11, rank: 0 }); + }); + + it('rejects an entries cursor', async () => { + const { service } = makeService(); + const cursor = encodeNotebookCursor({ scope: 'newest', key: '2026-02-03T04:05:06.123456Z', rank: 0, id: 1 }); + await expect(service.trash(user, { cursor })).rejects.toBeInstanceOf(BadRequestException); + }); +}); diff --git a/server/src/modules/notebook/notebook.service.ts b/server/src/modules/notebook/notebook.service.ts new file mode 100644 index 000000000..59c2bb2f9 --- /dev/null +++ b/server/src/modules/notebook/notebook.service.ts @@ -0,0 +1,296 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; + +import { + NOTEBOOK_BOOK_COLOR_MIX_MAX, + NOTEBOOK_BOOKS_PAGE_SIZE_DEFAULT, + NOTEBOOK_ON_THIS_DAY_MAX, + NOTEBOOK_PAGE_SIZE_DEFAULT, + NOTEBOOK_REVIEW_DAILY_SIZE, + NOTEBOOK_REVIEW_SHUFFLE_MAX, + normalizeCoverAspectRatio, + type AnnotationItem, + type NotebookBookLabel, + type NotebookBookRow, + type NotebookBooksResponse, + type NotebookEntriesResponse, + type NotebookEntry, + type NotebookEntryKind, + type NotebookHighlightEntry, + type NotebookJournalItem, + type NotebookOnThisDayResponse, + type NotebookOnThisDayYear, + type NotebookOverview, + type NotebookReviewResponse, + type NotebookTrashResponse, +} from '@bookorbit/types'; + +import type { RequestUser } from '../../common/types/request-user'; +import { buildSearchPattern } from '../../common/utils/accent-insensitive-search.utils'; +import { resolveTimeZone } from '../../common/utils/timezone.utils'; +import { BookCoverStore } from '../book-cover-store/book-cover-store.service'; +import { LibraryService } from '../library/library.service'; +import type { + NotebookBooksQueryDto, + NotebookEntriesQueryDto, + NotebookOnThisDayQueryDto, + NotebookOverviewQueryDto, + NotebookReviewQueryDto, + NotebookTrashQueryDto, +} from './dto/notebook-query.dto'; +import { decodeNotebookCursor, encodeNotebookCursor } from './notebook-cursor'; +import { isoFromSortKey, localYearOfSortKey, onThisDayRanges, rangeYearOfSortKey, reviewOlderThan } from './notebook-dates'; +import { NOTEBOOK_KIND_RANK, notebookKindForRank, resolveNotebookKinds, toNotebookFilters, type NotebookScope } from './notebook-filters'; +import { + toNotebookBookLabel, + toNotebookBookmark, + toNotebookHighlight, + toNotebookJournal, + toNotebookReview, + type NotebookMapOptions, +} from './notebook-mapper'; +import { NotebookRepository, type NotebookEntryKey } from './notebook.repository'; + +type EntryRef = Pick; +type LoadOptions = NotebookMapOptions & { status: 'active' | 'trashed' }; + +const TRASH_KINDS: NotebookEntryKind[] = ['highlight', 'journal']; +const OVERVIEW_ORIGINS = new Set(['web', 'koreader', 'kobo']); + +function entryMapKey(kind: NotebookEntryKind, id: number): string { + return `${kind}:${id}`; +} + +/** Drops a row that changed state between the keyset query and the load, so a listing never mixes trash and active. */ +function inStatus(deletedAt: Date | null, status: LoadOptions['status']): boolean { + return status === 'trashed' ? deletedAt != null : deletedAt == null; +} + +@Injectable() +export class NotebookService { + constructor( + private readonly repo: NotebookRepository, + private readonly libraryService: LibraryService, + private readonly coverStore: BookCoverStore, + ) {} + + async entries(user: RequestUser, query: NotebookEntriesQueryDto): Promise { + const sort = query.sort ?? 'newest'; + const cursor = query.cursor ? decodeNotebookCursor(query.cursor, sort) : undefined; + const limit = query.limit ?? NOTEBOOK_PAGE_SIZE_DEFAULT; + const filters = toNotebookFilters(query); + const kinds = resolveNotebookKinds(query.kinds, filters); + const scope = await this.scope(user); + if (scope.libraryIds.length === 0 || kinds.length === 0) return { items: [], books: {}, nextCursor: null }; + const keys = await this.repo.findEntryKeys(scope, { + kinds, + status: 'active', + filters, + sortColumn: sort === 'edited' ? 'edited' : 'written', + direction: sort === 'oldest' ? 'asc' : 'desc', + cursor, + limit: limit + 1, + }); + const page = keys.slice(0, limit); + const last = page.at(-1); + const nextCursor = + keys.length > limit && last ? encodeNotebookCursor({ scope: sort, key: last.sortKey, rank: last.kindRank, id: last.id }) : null; + const [items, books] = await Promise.all([this.loadEntries(user.id, page, { status: 'active' }), this.labels(page.map((key) => key.bookId))]); + return { items, books, nextCursor }; + } + + async overview(user: RequestUser, query: NotebookOverviewQueryDto): Promise { + const filters = toNotebookFilters(query); + const kinds = resolveNotebookKinds(undefined, filters); + const scope = await this.scope(user); + if (scope.libraryIds.length === 0) return emptyOverview(); + const row = await this.repo.findOverview(scope, filters, kinds); + return { + total: row.highlights + row.journal + row.bookmarks + row.reviews, + highlights: row.highlights, + journal: row.journal, + bookmarks: row.bookmarks, + reviews: row.reviews, + starred: row.starred, + withNotes: row.withNotes, + approximate: row.approximate, + books: row.books, + colors: row.colors, + origins: row.origins + .filter((entry) => OVERVIEW_ORIGINS.has(entry.origin)) + .map((entry) => ({ origin: entry.origin as AnnotationItem['origin'], count: entry.count })), + }; + } + + async books(user: RequestUser, query: NotebookBooksQueryDto): Promise { + const cursor = query.cursor ? decodeNotebookCursor(query.cursor, 'books') : undefined; + const limit = query.limit ?? NOTEBOOK_BOOKS_PAGE_SIZE_DEFAULT; + const term = query.q?.trim(); + const scope = await this.scope(user); + if (scope.libraryIds.length === 0) return { items: [], nextCursor: null }; + const rows = await this.repo.findBookActivity(scope, { searchPattern: term ? buildSearchPattern(term) : undefined, cursor, limit: limit + 1 }); + const page = rows.slice(0, limit); + const bookIds = page.map((row) => row.bookId); + const [latestKeys, colorRows, labels] = await Promise.all([ + this.repo.findLatestEntryPerBook(user.id, bookIds), + this.repo.findColorMix(user.id, bookIds), + this.labels(bookIds), + ]); + const latestEntries = await this.loadEntries(user.id, latestKeys, { status: 'active' }); + const latestByBook = new Map(latestEntries.map((entry) => [entry.bookId, entry])); + const colorsByBook = new Map(); + for (const row of colorRows) { + const mix = colorsByBook.get(row.bookId) ?? []; + if (mix.length < NOTEBOOK_BOOK_COLOR_MIX_MAX) mix.push({ color: row.color, count: Number(row.count) }); + colorsByBook.set(row.bookId, mix); + } + + const items: NotebookBookRow[] = []; + for (const row of page) { + const book = labels[String(row.bookId)]; + if (!book) continue; + items.push({ + book, + highlights: row.highlights, + journal: row.journal, + bookmarks: row.bookmarks, + hasReview: row.hasReview, + colors: colorsByBook.get(row.bookId) ?? [], + lastActivityAt: isoFromSortKey(row.lastKey), + latest: latestByBook.get(row.bookId) ?? null, + }); + } + const last = page.at(-1); + const nextCursor = rows.length > limit && last ? encodeNotebookCursor({ scope: 'books', key: last.lastKey, rank: 0, id: last.bookId }) : null; + return { items, nextCursor }; + } + + async review(user: RequestUser, query: NotebookReviewQueryDto): Promise { + const hasDate = query.date !== undefined; + const hasSeed = query.seed !== undefined; + if (hasDate === hasSeed) throw new BadRequestException('Pass exactly one of date and seed'); + const scope = await this.scope(user); + if (scope.libraryIds.length === 0) return { items: [], books: {} }; + const picks = + query.date !== undefined + ? await this.repo.findDailyReview(scope, { date: query.date, olderThan: reviewOlderThan(query.date), limit: NOTEBOOK_REVIEW_DAILY_SIZE }) + : await this.repo.findSeededReview(scope, { filters: toNotebookFilters(query), seed: query.seed!, limit: NOTEBOOK_REVIEW_SHUFFLE_MAX }); + const refs = picks.map((pick) => ({ kindRank: NOTEBOOK_KIND_RANK.highlight, id: pick.id })); + const [items, books] = await Promise.all([this.loadEntries(user.id, refs, { status: 'active' }), this.labels(picks.map((pick) => pick.bookId))]); + return { items: items as NotebookHighlightEntry[], books }; + } + + async onThisDay(user: RequestUser, query: NotebookOnThisDayQueryDto): Promise { + const timeZone = query.tz?.trim() || resolveTimeZone(user.settings?.timezone, 'UTC'); + const scope = await this.scope(user); + if (scope.libraryIds.length === 0) return { date: query.date, years: [], books: {} }; + const firstAt = await this.repo.findFirstEntryAt(user.id); + const ranges = onThisDayRanges(query.date, firstAt ? localYearOfSortKey(firstAt, timeZone) : null, timeZone); + const keys = await this.repo.findOnThisDayKeys(scope, { ranges, limit: NOTEBOOK_ON_THIS_DAY_MAX }); + const [entries, books] = await Promise.all([this.loadEntryMap(user.id, keys, { status: 'active' }), this.labels(keys.map((key) => key.bookId))]); + + const years: NotebookOnThisDayYear[] = []; + for (const key of keys) { + const entry = entries.get(entryMapKey(notebookKindForRank(key.kindRank), key.id)); + const year = rangeYearOfSortKey(key.sortKey, ranges); + if (!entry || year === null) continue; + const current = years.at(-1); + if (current?.year === year) current.items.push(entry); + else years.push({ year, items: [entry] }); + } + return { date: query.date, years, books }; + } + + async trash(user: RequestUser, query: NotebookTrashQueryDto): Promise { + const cursor = query.cursor ? decodeNotebookCursor(query.cursor, 'trash') : undefined; + const limit = query.limit ?? NOTEBOOK_PAGE_SIZE_DEFAULT; + const scope = await this.scope(user); + if (scope.libraryIds.length === 0) return { items: [], books: {}, nextCursor: null }; + const keys = await this.repo.findEntryKeys(scope, { + kinds: TRASH_KINDS, + status: 'trashed', + sortColumn: 'deleted', + direction: 'desc', + cursor, + limit: limit + 1, + }); + const page = keys.slice(0, limit); + const last = page.at(-1); + const nextCursor = + keys.length > limit && last ? encodeNotebookCursor({ scope: 'trash', key: last.sortKey, rank: last.kindRank, id: last.id }) : null; + const [items, books] = await Promise.all([ + this.loadEntries(user.id, page, { status: 'trashed', withDeletedAt: true }), + this.labels(page.map((key) => key.bookId)), + ]); + return { items: items as (NotebookHighlightEntry | NotebookJournalItem)[], books, nextCursor }; + } + + private async scope(user: RequestUser): Promise { + const libraryIds = await this.libraryService.findAccessibleLibraryIds(user); + return { userId: user.id, libraryIds, contentFilters: user.isSuperuser ? undefined : user.contentFilters }; + } + + private async loadEntries(userId: number, refs: readonly EntryRef[], options: LoadOptions): Promise { + const entries = await this.loadEntryMap(userId, refs, options); + return refs + .map((ref) => entries.get(entryMapKey(notebookKindForRank(ref.kindRank), ref.id))) + .filter((entry): entry is NotebookEntry => entry !== undefined); + } + + /** Loads each kind's rows for the page in one query per kind, keyed by kind and id. */ + private async loadEntryMap(userId: number, refs: readonly EntryRef[], options: LoadOptions): Promise> { + const idsOf = (kind: NotebookEntryKind) => [...new Set(refs.filter((ref) => ref.kindRank === NOTEBOOK_KIND_RANK[kind]).map((ref) => ref.id))]; + const [highlights, journal, bookmarks, reviews] = await Promise.all([ + this.repo.findHighlights(userId, idsOf('highlight')), + this.repo.findJournal(userId, idsOf('journal')), + this.repo.findBookmarks(userId, idsOf('bookmark')), + this.repo.findReviews(userId, idsOf('review')), + ]); + const entries = new Map(); + for (const row of highlights) { + if (inStatus(row.deletedAt, options.status)) entries.set(entryMapKey('highlight', row.id), toNotebookHighlight(row, options)); + } + for (const row of journal) { + if (inStatus(row.deletedAt, options.status)) entries.set(entryMapKey('journal', row.id), toNotebookJournal(row, options)); + } + for (const row of bookmarks) { + if (inStatus(row.deletedAt, options.status)) entries.set(entryMapKey('bookmark', row.id), toNotebookBookmark(row)); + } + for (const row of reviews) { + if (options.status === 'active') entries.set(entryMapKey('review', row.bookId), toNotebookReview(row)); + } + return entries; + } + + /** The label sidecar: one query for the books and one for their cover slots, whatever the page size. */ + private async labels(bookIds: readonly number[]): Promise> { + const ids = [...new Set(bookIds)]; + if (ids.length === 0) return {}; + const [rows, slotsByBook] = await Promise.all([this.repo.findLabels(ids), this.coverStore.slotsFor(ids)]); + const labels: Record = {}; + for (const row of rows) { + const coverVersion = this.coverStore.coverVersion( + normalizeCoverAspectRatio(row.coverAspectRatio), + slotsByBook.get(row.id) ?? [], + row.updatedAt.toISOString(), + ); + labels[String(row.id)] = toNotebookBookLabel(row, coverVersion); + } + return labels; + } +} + +function emptyOverview(): NotebookOverview { + return { + total: 0, + highlights: 0, + journal: 0, + bookmarks: 0, + reviews: 0, + starred: 0, + withNotes: 0, + approximate: 0, + books: 0, + colors: [], + origins: [], + }; +} diff --git a/server/test/notebook-hub.e2e-spec.ts b/server/test/notebook-hub.e2e-spec.ts new file mode 100644 index 000000000..a286d26c4 --- /dev/null +++ b/server/test/notebook-hub.e2e-spec.ts @@ -0,0 +1,507 @@ +import { randomUUID } from 'crypto'; +import { eq, sql } from 'drizzle-orm'; + +import type { + NotebookBooksResponse, + NotebookEntriesResponse, + NotebookEntry, + NotebookOnThisDayResponse, + NotebookOverview, + NotebookReviewResponse, + NotebookTrashResponse, +} from '@bookorbit/types'; + +import * as schema from '../src/db/schema'; +import { createEpubFixture } from './e2e/reader-state-isolation/reader-state-isolation-fixture-builder'; +import { + authHeader, + closeReaderStateIsolationE2EContext, + createLibraryWithFolder, + createReaderStateIsolationE2EContext, + createUserAndLogin, + grantLibraryAccess, + locateBookByAbsolutePath, + triggerAndWaitForLibraryScan, + type LocatedBookFile, + type ReaderStateIsolationE2EContext, + type TestUserSession, +} from './e2e/reader-state-isolation/reader-state-isolation-harness'; + +/** Two entries of different kinds, and two of the same kind, written in the same microsecond. */ +const TIE = '2026-03-03T12:00:00.500000Z'; + +function at(iso: string) { + return sql`${iso}::timestamptz`; +} + +describe('Notebook hub (e2e)', { timeout: 180_000 }, () => { + let ctx!: ReaderStateIsolationE2EContext; + let bookA!: LocatedBookFile; + let bookB!: LocatedBookFile; + let bookC!: LocatedBookFile; + let owner!: TestUserSession; + let outsider!: TestUserSession; + const ids: Record = {}; + + async function highlight( + name: string, + book: LocatedBookFile, + values: { + text: string; + color?: string; + origin?: 'web' | 'koreader' | 'kobo'; + note?: string; + createdAt: string; + sourceCreatedAt?: string; + starred?: boolean; + deletedAt?: string; + cfiStatus?: 'exact' | 'repaired'; + }, + ) { + const [row] = await ctx.db + .insert(schema.annotations) + .values({ + userId: owner.userId, + bookId: book.bookId, + text: values.text, + color: values.color ?? '#FACC15', + style: 'highlight', + origin: values.origin ?? 'web', + note: values.note ?? null, + createdAt: at(values.createdAt), + updatedAt: at(values.createdAt), + sourceCreatedAt: values.sourceCreatedAt ? at(values.sourceCreatedAt) : null, + starredAt: values.starred ? at(values.createdAt) : null, + deletedAt: values.deletedAt ? at(values.deletedAt) : null, + }) + .returning({ id: schema.annotations.id }); + if (values.cfiStatus) { + await ctx.db.insert(schema.annotationPositions).values({ + annotationId: row.id, + userId: owner.userId, + bookFileId: book.bookFileId, + format: 'cfi', + pos0: 'epubcfi(/6/2!/4/2/1:0)', + status: values.cfiStatus, + }); + } + ids[name] = `highlight:${row.id}`; + } + + async function journal(name: string, book: LocatedBookFile, body: string, createdAt: string, deletedAt?: string) { + const [row] = await ctx.db + .insert(schema.bookJournalEntries) + .values({ + clientId: randomUUID(), + userId: owner.userId, + bookId: book.bookId, + body, + createdAt: at(createdAt), + updatedAt: at(createdAt), + deletedAt: deletedAt ? at(deletedAt) : null, + }) + .returning({ id: schema.bookJournalEntries.id }); + ids[name] = `journal:${row.id}`; + } + + async function bookmark( + name: string, + book: LocatedBookFile, + values: { title: string; note?: string; origin?: 'web' | 'koreader'; createdAt: string; positionSeconds?: number }, + ) { + const [row] = await ctx.db + .insert(schema.bookmarks) + .values({ + userId: owner.userId, + bookId: book.bookId, + title: values.title, + note: values.note ?? null, + origin: values.origin ?? 'web', + cfi: values.positionSeconds === undefined ? `epubcfi(/6/4!/4/2/${Object.keys(ids).length})` : null, + positionSeconds: values.positionSeconds ?? null, + createdAt: at(values.createdAt), + updatedAt: at(values.createdAt), + }) + .returning({ id: schema.bookmarks.id }); + ids[name] = `bookmark:${row.id}`; + } + + async function nameBook(book: LocatedBookFile, title: string, author: string) { + await ctx.db + .insert(schema.bookMetadata) + .values({ bookId: book.bookId, title }) + .onConflictDoUpdate({ target: schema.bookMetadata.bookId, set: { title } }); + const [created] = await ctx.db + .insert(schema.authors) + // A digits-only suffix keeps the name unique across reruns without ever matching a text search below. + .values({ name: `${author} ${Date.now()}${Math.floor(Math.random() * 1000)}` }) + .returning({ id: schema.authors.id }); + await ctx.db.delete(schema.bookAuthors).where(eq(schema.bookAuthors.bookId, book.bookId)); + await ctx.db.insert(schema.bookAuthors).values({ bookId: book.bookId, authorId: created.id, displayOrder: 0 }); + } + + function get(user: TestUserSession, url: string) { + return ctx.app.inject({ method: 'GET', url: `/api/v1${url}`, headers: authHeader(user.accessToken) }).then((response) => ({ + status: response.statusCode, + body: response.json() as T, + })); + } + + function names(items: NotebookEntry[]): string[] { + const byRef = new Map(Object.entries(ids).map(([name, ref]) => [ref, name])); + return items.map((item) => byRef.get(`${item.kind}:${item.id}`) ?? `${item.kind}:${item.id}`); + } + + async function walk(url: string, limit: number): Promise { + const seen: string[] = []; + let cursor: string | null = null; + for (let page = 0; page < 50; page++) { + const separator = url.includes('?') ? '&' : '?'; + const { status, body } = await get( + owner, + `${url}${separator}limit=${limit}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, + ); + expect(status).toBe(200); + expect(body.items.length).toBeLessThanOrEqual(limit); + seen.push(...names(body.items)); + cursor = body.nextCursor; + if (!cursor) return seen; + } + throw new Error('Paging did not terminate'); + } + + beforeAll(async () => { + ctx = await createReaderStateIsolationE2EContext(); + const library = await createLibraryWithFolder(ctx, { name: `notebook-hub-${randomUUID()}` }); + const hidden = await createLibraryWithFolder(ctx, { name: `notebook-hub-hidden-${randomUUID()}` }); + const pathA = await createEpubFixture(library.folderPath, 'hub-a.epub', { title: `Hub A ${randomUUID()}`, uid: `urn:uuid:${randomUUID()}` }); + const pathB = await createEpubFixture(library.folderPath, 'hub-b.epub', { title: `Hub B ${randomUUID()}`, uid: `urn:uuid:${randomUUID()}` }); + const pathC = await createEpubFixture(hidden.folderPath, 'hub-c.epub', { title: `Hub C ${randomUUID()}`, uid: `urn:uuid:${randomUUID()}` }); + await triggerAndWaitForLibraryScan(ctx, library.libraryId); + await triggerAndWaitForLibraryScan(ctx, hidden.libraryId); + bookA = await locateBookByAbsolutePath(ctx, pathA); + bookB = await locateBookByAbsolutePath(ctx, pathB); + bookC = await locateBookByAbsolutePath(ctx, pathC); + + owner = await createUserAndLogin(ctx); + outsider = await createUserAndLogin(ctx); + await grantLibraryAccess(ctx, owner.userId, library.libraryId, 'viewer'); + + await nameBook(bookA, 'Dune Messiah', 'Frank Herbert'); + await nameBook(bookB, 'Café Notes', 'Zoë Writer'); + + await highlight('hA1', bookA, { text: 'The spice must flow', createdAt: '2026-03-01T10:00:00.000001Z', cfiStatus: 'exact' }); + await highlight('hA2', bookA, { + text: 'Fear is the mind-killer, 100% of the time', + color: '#38BDF8', + origin: 'kobo', + note: 'remember', + createdAt: '2026-03-05T00:00:00.000000Z', + sourceCreatedAt: '2026-03-02T09:00:00.000002Z', + starred: true, + cfiStatus: 'repaired', + }); + await highlight('hB1', bookB, { text: 'Plain 1000 words', origin: 'koreader', createdAt: TIE }); + await highlight('hB2', bookB, { text: 'Second at the same instant', color: '#4ADE80', createdAt: TIE, cfiStatus: 'exact' }); + await journal('jB1', bookB, 'Journal thought', TIE); + await bookmark('bmB1', bookB, { title: 'At 2:00', note: 'listen again', origin: 'koreader', createdAt: TIE, positionSeconds: 120 }); + await bookmark('bmA1', bookA, { title: 'Chapter 2', note: ' ', createdAt: '2026-03-04T08:00:00.000003Z' }); + await ctx.db + .insert(schema.userBookNotes) + .values({ userId: owner.userId, bookId: bookA.bookId, note: 'A classic.', updatedAt: at('2026-03-06T00:00:00.000000Z') }); + await ctx.db.insert(schema.userBookRatings).values({ userId: owner.userId, bookId: bookA.bookId, rating: 5 }); + await ctx.db + .insert(schema.userBookNotes) + .values({ userId: owner.userId, bookId: bookB.bookId, note: ' \n ', updatedAt: at('2026-03-09T00:00:00.000000Z') }); + ids.rA = `review:${bookA.bookId}`; + + // 2025-10-04 01:30 in Berlin, and 2025-10-05 00:30 in Berlin. + await highlight('hA4', bookA, { text: 'Berlin morning', createdAt: '2025-10-03T23:30:00.000000Z', cfiStatus: 'exact' }); + await highlight('hA5', bookA, { text: 'UTC evening', createdAt: '2025-10-04T22:30:00.000000Z', cfiStatus: 'exact' }); + await journal('jA3', bookA, 'Two years ago', '2024-10-04T12:00:00.000000Z'); + + await highlight('hA3', bookA, { text: 'Trashed highlight', createdAt: '2026-02-01T00:00:00.000000Z', deletedAt: '2026-03-07T00:00:00.000000Z' }); + await journal('jA2', bookA, 'Trashed thought', '2026-02-02T00:00:00.000000Z', '2026-03-08T00:00:00.000000Z'); + + const [hidden1] = await ctx.db + .insert(schema.annotations) + .values({ userId: owner.userId, bookId: bookC.bookId, text: 'Out of reach', createdAt: at('2026-03-10T00:00:00.000000Z') }) + .returning({ id: schema.annotations.id }); + ids.hC1 = `highlight:${hidden1.id}`; + }, 180_000); + + afterAll(async () => { + if (ctx) await closeReaderStateIsolationE2EContext(ctx); + }); + + const NEWEST = ['rA', 'bmA1', 'bmB1', 'jB1', 'hB2', 'hB1', 'hA2', 'hA1', 'hA5', 'hA4', 'jA3']; + + it('advertises notebook-hub on app-info', async () => { + const { body } = await get<{ features: string[] }>(owner, '/app-info'); + expect(body.features).toContain('notebook-hub'); + }); + + describe('GET /notebook/entries', () => { + it('merges every kind newest first, breaking ties on kind and then id, and only from libraries the reader can open', async () => { + const { status, body } = await get(owner, '/notebook/entries'); + expect(status).toBe(200); + expect(names(body.items)).toEqual(NEWEST); + expect(body.nextCursor).toBeNull(); + expect(Object.keys(body.books).sort()).toEqual([String(bookA.bookId), String(bookB.bookId)].sort()); + }); + + it('returns the contract shape for each kind and the book labels', async () => { + const { body } = await get(owner, '/notebook/entries'); + const byName = new Map(names(body.items).map((name, index) => [name, body.items[index]!])); + expect(byName.get('hA2')).toEqual({ + kind: 'highlight', + id: expect.any(Number), + bookId: bookA.bookId, + clientId: null, + createdAt: '2026-03-02T09:00:00.000Z', + updatedAt: '2026-03-05T00:00:00.000Z', + text: 'Fear is the mind-killer, 100% of the time', + color: '#38BDF8', + style: 'highlight', + note: 'remember', + chapterTitle: null, + cfi: 'epubcfi(/6/2!/4/2/1:0)', + positionStatus: 'repaired', + origin: 'kobo', + starredAt: '2026-03-05T00:00:00.000Z', + }); + expect(byName.get('hB1')).toMatchObject({ cfi: null, positionStatus: null }); + expect(byName.get('jB1')).toMatchObject({ kind: 'journal', clientId: expect.any(String), body: 'Journal thought', positionPercent: null }); + expect(byName.get('bmB1')).toMatchObject({ kind: 'bookmark', title: 'At 2:00', cfi: null, positionSeconds: 120, origin: 'koreader' }); + expect(byName.get('rA')).toEqual({ + kind: 'review', + id: bookA.bookId, + bookId: bookA.bookId, + clientId: null, + createdAt: '2026-03-06T00:00:00.000Z', + updatedAt: '2026-03-06T00:00:00.000Z', + body: 'A classic.', + rating: 5, + }); + for (const item of body.items) expect(item).not.toHaveProperty('deletedAt'); + + expect(body.books[String(bookA.bookId)]).toEqual({ + id: bookA.bookId, + title: 'Dune Messiah', + author: expect.stringMatching(/^Frank Herbert /), + hasCover: expect.any(Boolean), + coverVersion: expect.any(String), + coverAspectRatio: '2/3', + readFileId: bookA.bookFileId, + readFileFormat: 'epub', + hasAudio: false, + }); + }); + + it.each([1, 2, 3, 4])('pages with limit %i to exactly the unpaged list', async (limit) => { + expect(await walk('/notebook/entries', limit)).toEqual(NEWEST); + expect(await walk('/notebook/entries?sort=oldest', limit)).toEqual([...NEWEST].reverse()); + }); + + it('rejects a cursor from another sort and a malformed one', async () => { + const { body } = await get(owner, '/notebook/entries?limit=2'); + expect((await get(owner, `/notebook/entries?sort=oldest&cursor=${body.nextCursor}`)).status).toBe(400); + expect((await get(owner, '/notebook/entries?cursor=not-a-cursor')).status).toBe(400); + expect((await get(owner, '/notebook/entries?kinds=highlight,podcast')).status).toBe(400); + expect((await get(owner, '/notebook/entries?limit=101')).status).toBe(400); + }); + + it.each([ + ['starred=true', ['hA2']], + ['starred=false', NEWEST], + ['colors=%2338BDF8,%234ADE80', ['hB2', 'hA2']], + ['approximate=true', ['hB1', 'hA2']], + ['hasNote=true', ['rA', 'bmB1', 'jB1', 'hA2', 'jA3']], + ['origins=kobo', ['hA2']], + ['origins=koreader', ['bmB1', 'hB1']], + ['origins=web', ['rA', 'bmA1', 'jB1', 'hB2', 'hA1', 'hA5', 'hA4', 'jA3']], + ['kinds=journal,review', ['rA', 'jB1', 'jA3']], + ['kinds=journal&starred=true', []], + ['q=100%25', ['hA2']], + ['q=%25', ['hA2']], + ['q=MIND-killer', ['hA2']], + ['q=herbert', ['rA', 'bmA1', 'hA2', 'hA1', 'hA5', 'hA4', 'jA3']], + ['q=cafe', ['bmB1', 'jB1', 'hB2', 'hB1']], + ['q=zoe', ['bmB1', 'jB1', 'hB2', 'hB1']], + ])('filters %s', async (query, expected) => { + const { status, body } = await get(owner, `/notebook/entries?${query}`); + expect(status).toBe(200); + expect(names(body.items)).toEqual(expected); + }); + + it('filters by book and sorts by edit time', async () => { + const { body } = await get(owner, `/notebook/entries?bookId=${bookB.bookId}`); + expect(names(body.items)).toEqual(['bmB1', 'jB1', 'hB2', 'hB1']); + const edited = await get(owner, '/notebook/entries?sort=edited&kinds=highlight&limit=2'); + expect(names(edited.body.items)).toEqual(['hA2', 'hB2']); + }); + + it('is empty for a reader with no library access', async () => { + const { status, body } = await get(outsider, '/notebook/entries'); + expect(status).toBe(200); + expect(body).toEqual({ items: [], books: {}, nextCursor: null }); + }); + }); + + describe('GET /notebook/overview', () => { + it('counts every kind under the same scope', async () => { + const { status, body } = await get(owner, '/notebook/overview'); + expect(status).toBe(200); + expect(body).toEqual({ + total: 11, + highlights: 6, + journal: 2, + bookmarks: 2, + reviews: 1, + starred: 1, + withNotes: 5, + approximate: 2, + books: 2, + colors: [ + { color: '#FACC15', count: 4 }, + { color: '#38BDF8', count: 1 }, + { color: '#4ADE80', count: 1 }, + ], + origins: [ + { origin: 'web', count: 4 }, + { origin: 'kobo', count: 1 }, + { origin: 'koreader', count: 1 }, + ], + }); + }); + + it('honours the filters and rejects kinds', async () => { + const starred = await get(owner, '/notebook/overview?starred=true'); + expect(starred.body).toMatchObject({ total: 1, highlights: 1, journal: 0, bookmarks: 0, reviews: 0, books: 1 }); + expect((await get(owner, '/notebook/overview?kinds=highlight')).status).toBe(400); + }); + + it('is all zeros for a reader with no library access', async () => { + const { body } = await get(outsider, '/notebook/overview'); + expect(body).toMatchObject({ total: 0, books: 0, colors: [], origins: [] }); + }); + }); + + describe('GET /notebook/books', () => { + it('lists books by last activity with counts, colours and the latest entry, paged by cursor', async () => { + const first = await get(owner, '/notebook/books?limit=1'); + expect(first.status).toBe(200); + expect(first.body.items).toHaveLength(1); + expect(first.body.items[0]).toMatchObject({ + book: { id: bookA.bookId, title: 'Dune Messiah' }, + highlights: 4, + journal: 1, + bookmarks: 1, + hasReview: true, + colors: [ + { color: '#FACC15', count: 3 }, + { color: '#38BDF8', count: 1 }, + ], + lastActivityAt: '2026-03-06T00:00:00.000Z', + latest: { kind: 'review', id: bookA.bookId }, + }); + + const second = await get(owner, `/notebook/books?limit=1&cursor=${first.body.nextCursor}`); + expect(second.body.items[0]).toMatchObject({ + book: { id: bookB.bookId, title: 'Café Notes' }, + highlights: 2, + journal: 1, + bookmarks: 1, + hasReview: false, + lastActivityAt: '2026-03-03T12:00:00.500Z', + latest: { kind: 'bookmark' }, + }); + expect(second.body.nextCursor).toBeNull(); + }); + + it('matches title or author and is empty without access', async () => { + const byAuthor = await get(owner, '/notebook/books?q=zoe'); + expect(byAuthor.body.items.map((row) => row.book.id)).toEqual([bookB.bookId]); + const outside = await get(outsider, '/notebook/books'); + expect(outside.body).toEqual({ items: [], nextCursor: null }); + }); + }); + + describe('GET /notebook/review', () => { + it('builds a daily deck, one highlight per book with starred first, the same all day', async () => { + const first = await get(owner, '/notebook/review?date=2026-10-04'); + const again = await get(owner, '/notebook/review?date=2026-10-04'); + expect(first.status).toBe(200); + expect(again.body).toEqual(first.body); + expect(first.body.items).toHaveLength(2); + expect(names(first.body.items)[0]).toBe('hA2'); + expect(new Set(first.body.items.map((item) => item.bookId)).size).toBe(2); + expect(Object.keys(first.body.books).sort()).toEqual([String(bookA.bookId), String(bookB.bookId)].sort()); + }); + + it('shuffles the filtered highlights the same way for the same seed', async () => { + const first = await get(owner, '/notebook/review?seed=7'); + const again = await get(owner, '/notebook/review?seed=7'); + expect(names(again.body.items)).toEqual(names(first.body.items)); + expect([...names(first.body.items)].sort()).toEqual(['hA1', 'hA2', 'hA4', 'hA5', 'hB1', 'hB2']); + const starred = await get(owner, '/notebook/review?seed=7&starred=true'); + expect(names(starred.body.items)).toEqual(['hA2']); + }); + + it('requires exactly one of date and seed', async () => { + expect((await get(owner, '/notebook/review')).status).toBe(400); + expect((await get(owner, '/notebook/review?date=2026-10-04&seed=1')).status).toBe(400); + expect((await get(owner, '/notebook/review?date=2026-02-30')).status).toBe(400); + }); + }); + + describe('GET /notebook/on-this-day', () => { + it('finds earlier years on the local day in the requested zone', async () => { + const berlin = await get(owner, '/notebook/on-this-day?date=2026-10-04&tz=Europe/Berlin'); + expect(berlin.status).toBe(200); + expect(berlin.body.date).toBe('2026-10-04'); + expect(berlin.body.years.map((year) => [year.year, names(year.items)])).toEqual([ + [2025, ['hA4']], + [2024, ['jA3']], + ]); + expect(Object.keys(berlin.body.books)).toEqual([String(bookA.bookId)]); + + const utc = await get(owner, '/notebook/on-this-day?date=2026-10-04&tz=UTC'); + expect(utc.body.years.map((year) => [year.year, names(year.items)])).toEqual([ + [2025, ['hA5']], + [2024, ['jA3']], + ]); + }); + + it("defaults to the reader's timezone setting", async () => { + await ctx.db + .update(schema.users) + .set({ settings: { timezone: 'Europe/Berlin' } }) + .where(eq(schema.users.id, owner.userId)); + const { body } = await get(owner, '/notebook/on-this-day?date=2026-10-04'); + expect(body.years.map((year) => [year.year, names(year.items)])).toEqual([ + [2025, ['hA4']], + [2024, ['jA3']], + ]); + }); + + it('validates the date and zone', async () => { + expect((await get(owner, '/notebook/on-this-day')).status).toBe(400); + expect((await get(owner, '/notebook/on-this-day?date=2026-10-04&tz=Mars/Olympus')).status).toBe(400); + }); + }); + + describe('GET /notebook/trash', () => { + it('lists deleted highlights and journal entries by deletion time, with deletedAt', async () => { + const first = await get(owner, '/notebook/trash?limit=1'); + expect(first.status).toBe(200); + expect(names(first.body.items)).toEqual(['jA2']); + expect(first.body.items[0]).toMatchObject({ kind: 'journal', deletedAt: '2026-03-08T00:00:00.000Z' }); + const second = await get(owner, `/notebook/trash?limit=1&cursor=${first.body.nextCursor}`); + expect(names(second.body.items)).toEqual(['hA3']); + expect(second.body.items[0]).toMatchObject({ kind: 'highlight', deletedAt: '2026-03-07T00:00:00.000Z' }); + expect(second.body.nextCursor).toBeNull(); + expect((await get(outsider, '/notebook/trash')).body).toEqual({ items: [], books: {}, nextCursor: null }); + }); + }); +});