Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 08:21:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: improve drizzle queries

Browse Source
This commit is contained in:
neon
2026-04-09 05:14:28 -06:00
parent b0ed068fd6
commit 02a0d711c3
33 changed files with 1665 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/common/constants/pagination.constants.ts
+5
View File
@@ -0,0 +1,5 @@
export const MAX_OFFSET_ROWS = 50_000;
export function isOffsetWithinLimit(offset: number): boolean {
return offset <= MAX_OFFSET_ROWS;
}
server/src/db/migrations/0000_baseline.sql
+10
View File
@@ -901,6 +901,7 @@ CREATE INDEX "password_reset_tokens_user_id_idx" ON "password_reset_tokens" USIN
CREATE INDEX "password_reset_tokens_expires_at_idx" ON "password_reset_tokens" USING btree ("expires_at");--> statement-breakpoint
CREATE INDEX "refresh_tokens_user_id_idx" ON "refresh_tokens" USING btree ("user_id");--> statement-breakpoint
CREATE INDEX "refresh_tokens_expires_at_idx" ON "refresh_tokens" USING btree ("expires_at");--> statement-breakpoint
CREATE INDEX "user_library_access_library_user_idx" ON "user_library_access" USING btree ("library_id","user_id");--> statement-breakpoint
CREATE UNIQUE INDEX "users_username_lower_uidx" ON "users" USING btree (lower("username"));--> statement-breakpoint
CREATE UNIQUE INDEX "users_email_lower_uidx" ON "users" USING btree (lower("email")) WHERE "users"."email" is not null;--> statement-breakpoint
CREATE UNIQUE INDEX "users_oidc_subject_issuer_uidx" ON "users" USING btree ("oidc_subject","oidc_issuer") WHERE "users"."oidc_subject" is not null and "users"."oidc_issuer" is not null;--> statement-breakpoint
@@ -938,14 +939,21 @@ CREATE INDEX "author_enrichment_queue_next_attempt_idx" ON "author_enrichment_qu
CREATE UNIQUE INDEX "author_enrichment_queue_single_processing_idx" ON "author_enrichment_queue" USING btree ("status") WHERE "author_enrichment_queue"."status" = 'processing';--> statement-breakpoint
CREATE INDEX "authors_name_trgm_idx" ON "authors" USING gin ("name" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "book_authors_author_id_idx" ON "book_authors" USING btree ("author_id");--> statement-breakpoint
CREATE INDEX "book_genres_genre_id_idx" ON "book_genres" USING btree ("genre_id");--> statement-breakpoint
CREATE INDEX "bm_title_trgm_idx" ON "book_metadata" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "bm_title_lower_idx" ON "book_metadata" USING btree (lower("title"));--> statement-breakpoint
CREATE INDEX "bm_series_trgm_idx" ON "book_metadata" USING gin ("series_name" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "bm_publisher_trgm_idx" ON "book_metadata" USING gin ("publisher" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "bm_language_idx" ON "book_metadata" USING btree ("language");--> statement-breakpoint
CREATE INDEX "bm_published_year_idx" ON "book_metadata" USING btree ("published_year");--> statement-breakpoint
CREATE INDEX "bm_series_name_index_idx" ON "book_metadata" USING btree ("series_name","series_index");--> statement-breakpoint
CREATE INDEX "bm_isbn10_idx" ON "book_metadata" USING btree ("isbn10");--> statement-breakpoint
CREATE INDEX "bm_isbn13_idx" ON "book_metadata" USING btree ("isbn13");--> statement-breakpoint
CREATE INDEX "bm_embedding_hnsw_cosine_idx" ON "book_metadata" USING hnsw ("embedding" vector_cosine_ops);--> statement-breakpoint
CREATE INDEX "bmfq_status_idx" ON "book_metadata_fetch_queue" USING btree ("status");--> statement-breakpoint
CREATE INDEX "bmfq_created_at_idx" ON "book_metadata_fetch_queue" USING btree ("created_at");--> statement-breakpoint
CREATE INDEX "bmfq_status_created_book_idx" ON "book_metadata_fetch_queue" USING btree ("status","created_at","book_id");--> statement-breakpoint
CREATE INDEX "book_tags_tag_id_idx" ON "book_tags" USING btree ("tag_id");--> statement-breakpoint
CREATE INDEX "book_narrators_narrator_id_idx" ON "book_narrators" USING btree ("narrator_id");--> statement-breakpoint
CREATE INDEX "narrators_name_trgm_idx" ON "narrators" USING gin ("name" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "scan_jobs_library_status_idx" ON "scan_jobs" USING btree ("library_id","status");--> statement-breakpoint
@@ -958,6 +966,7 @@ CREATE UNIQUE INDEX "bookmarks_user_book_pos_uidx" ON "bookmarks" USING btree ("
CREATE UNIQUE INDEX "rdp_user_format_idx" ON "reader_default_preferences" USING btree ("user_id","format_group");--> statement-breakpoint
CREATE UNIQUE INDEX "rp_user_file_idx" ON "reader_preferences" USING btree ("user_id","book_file_id");--> statement-breakpoint
CREATE INDEX "reading_progress_user_id_idx" ON "reading_progress" USING btree ("user_id");--> statement-breakpoint
CREATE INDEX "reading_progress_user_updated_at_idx" ON "reading_progress" USING btree ("user_id","updated_at");--> statement-breakpoint
CREATE UNIQUE INDEX "rs_user_session_id_uidx" ON "reading_sessions" USING btree ("user_id","session_id");--> statement-breakpoint
CREATE INDEX "rs_user_started_at_idx" ON "reading_sessions" USING btree ("user_id","started_at");--> statement-breakpoint
CREATE INDEX "rs_book_file_started_at_idx" ON "reading_sessions" USING btree ("book_file_id","started_at");--> statement-breakpoint
@@ -971,6 +980,7 @@ CREATE INDEX "oidc_sessions_sid_idx" ON "oidc_sessions" USING btree ("oidc_sessi
CREATE INDEX "oidc_sessions_expires_at_idx" ON "oidc_sessions" USING btree ("expires_at");--> statement-breakpoint
CREATE UNIQUE INDEX "opds_users_username_lower_uidx" ON "opds_users" USING btree (lower("username"));--> statement-breakpoint
CREATE INDEX "kobo_devices_user_id_idx" ON "kobo_devices" USING btree ("user_id");--> statement-breakpoint
CREATE INDEX "kobo_snapshot_books_snapshot_synced_book_idx" ON "kobo_snapshot_books" USING btree ("snapshot_id","synced","book_id");--> statement-breakpoint
CREATE INDEX "book_bucket_files_status_idx" ON "book_bucket_files" USING btree ("status");--> statement-breakpoint
CREATE INDEX "fwl_book_id_written_at_idx" ON "file_write_log" USING btree ("book_id","written_at");--> statement-breakpoint
CREATE UNIQUE INDEX "email_templates_one_default_per_user_uidx" ON "email_templates" USING btree ("user_id") WHERE "email_templates"."is_default" = true and "email_templates"."user_id" is not null;--> statement-breakpoint
server/src/db/migrations/meta/0000_snapshot.json
+195 -5
View File
@@ -1,5 +1,5 @@
{
"id": "48886cdb-ff31-4dca-9781-e0a35a0750c8",
"id": "3d7638fa-4c77-4988-a03b-15e20345198d",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
@@ -443,7 +443,29 @@
"default": "'viewer'"
}
},
"indexes": {},
"indexes": {
"user_library_access_library_user_idx": {
"name": "user_library_access_library_user_idx",
"columns": [
{
"expression": "library_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"user_library_access_user_id_users_id_fk": {
"name": "user_library_access_user_id_users_id_fk",
@@ -3106,7 +3128,23 @@
"notNull": true
}
},
"indexes": {},
"indexes": {
"book_genres_genre_id_idx": {
"name": "book_genres_genre_id_idx",
"columns": [
{
"expression": "genre_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"book_genres_book_id_books_id_fk": {
"name": "book_genres_book_id_books_id_fk",
@@ -3349,6 +3387,21 @@
"method": "gin",
"with": {}
},
"bm_title_lower_idx": {
"name": "bm_title_lower_idx",
"columns": [
{
"expression": "lower(\"title\")",
"asc": true,
"isExpression": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"bm_series_trgm_idx": {
"name": "bm_series_trgm_idx",
"columns": [
@@ -3431,6 +3484,51 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"bm_isbn10_idx": {
"name": "bm_isbn10_idx",
"columns": [
{
"expression": "isbn10",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"bm_isbn13_idx": {
"name": "bm_isbn13_idx",
"columns": [
{
"expression": "isbn13",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"bm_embedding_hnsw_cosine_idx": {
"name": "bm_embedding_hnsw_cosine_idx",
"columns": [
{
"expression": "\"embedding\" vector_cosine_ops",
"asc": true,
"isExpression": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "hnsw",
"with": {}
}
},
"foreignKeys": {
@@ -3565,6 +3663,33 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"bmfq_status_created_book_idx": {
"name": "bmfq_status_created_book_idx",
"columns": [
{
"expression": "status",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "book_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -3614,7 +3739,23 @@
"notNull": true
}
},
"indexes": {},
"indexes": {
"book_tags_tag_id_idx": {
"name": "book_tags_tag_id_idx",
"columns": [
{
"expression": "tag_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"book_tags_book_id_books_id_fk": {
"name": "book_tags_book_id_books_id_fk",
@@ -4609,6 +4750,27 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"reading_progress_user_updated_at_idx": {
"name": "reading_progress_user_updated_at_idx",
"columns": [
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -5672,7 +5834,35 @@
"notNull": false
}
},
"indexes": {},
"indexes": {
"kobo_snapshot_books_snapshot_synced_book_idx": {
"name": "kobo_snapshot_books_snapshot_synced_book_idx",
"columns": [
{
"expression": "snapshot_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "synced",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "book_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"kobo_snapshot_books_snapshot_id_kobo_library_snapshots_id_fk": {
"name": "kobo_snapshot_books_snapshot_id_kobo_library_snapshots_id_fk",
server/src/db/migrations/meta/_journal.json
+1 -1
View File
@@ -5,7 +5,7 @@
{
"idx": 0,
"version": "7",
"when": 1775495173768,
"when": 1775770381607,
"tag": "0000_baseline",
"breakpoints": true
}
server/src/db/schema/auth.ts
+1 -1
View File
@@ -82,7 +82,7 @@ export const userLibraryAccess = pgTable(
.references(() => libraries.id, { onDelete: 'cascade' }),
accessLevel: libraryAccessLevelEnum('access_level').notNull().default('viewer'),
},
(t) => [primaryKey({ columns: [t.userId, t.libraryId] })],
(t) => [primaryKey({ columns: [t.userId, t.libraryId] }), index('user_library_access_library_user_idx').on(t.libraryId, t.userId)],
);
export const refreshTokens = pgTable(
server/src/db/schema/kobo.ts
+4 -1
View File
@@ -75,7 +75,10 @@ export const koboSnapshotBooks = pgTable(
fileHash: varchar('file_hash', { length: 64 }),
metadataHash: varchar('metadata_hash', { length: 64 }),
},
(t) => [primaryKey({ columns: [t.snapshotId, t.bookId] })],
(t) => [
primaryKey({ columns: [t.snapshotId, t.bookId] }),
index('kobo_snapshot_books_snapshot_synced_book_idx').on(t.snapshotId, t.synced, t.bookId),
],
);
export const koboReadingStates = pgTable(
server/src/db/schema/metadata.ts
+7 -2
View File
@@ -76,11 +76,15 @@ export const bookMetadata = pgTable(
},
(t) => [
index('bm_title_trgm_idx').using('gin', t.title.op('gin_trgm_ops')),
index('bm_title_lower_idx').on(sql`lower(${t.title})`),
index('bm_series_trgm_idx').using('gin', t.seriesName.op('gin_trgm_ops')),
index('bm_publisher_trgm_idx').using('gin', t.publisher.op('gin_trgm_ops')),
index('bm_language_idx').on(t.language),
index('bm_published_year_idx').on(t.publishedYear),
index('bm_series_name_index_idx').on(t.seriesName, t.seriesIndex),
index('bm_isbn10_idx').on(t.isbn10),
index('bm_isbn13_idx').on(t.isbn13),
index('bm_embedding_hnsw_cosine_idx').using('hnsw', sql`${t.embedding} vector_cosine_ops`),
check('book_metadata_rating_range_chk', sql`${t.rating} is null or (${t.rating} >= 1 and ${t.rating} <= 10)`),
check('book_metadata_published_year_range_chk', sql`${t.publishedYear} is null or (${t.publishedYear} >= 1000 and ${t.publishedYear} <= 2200)`),
check('book_metadata_page_count_nonnegative_chk', sql`${t.pageCount} is null or ${t.pageCount} >= 0`),
@@ -166,7 +170,7 @@ export const bookGenres = pgTable(
.notNull()
.references(() => genres.id, { onDelete: 'cascade' }),
},
(t) => [primaryKey({ columns: [t.bookId, t.genreId] })],
(t) => [primaryKey({ columns: [t.bookId, t.genreId] }), index('book_genres_genre_id_idx').on(t.genreId)],
);
export const tags = pgTable('tags', {
@@ -184,7 +188,7 @@ export const bookTags = pgTable(
.notNull()
.references(() => tags.id, { onDelete: 'cascade' }),
},
(t) => [primaryKey({ columns: [t.bookId, t.tagId] })],
(t) => [primaryKey({ columns: [t.bookId, t.tagId] }), index('book_tags_tag_id_idx').on(t.tagId)],
);
export type BookMetadata = typeof bookMetadata.$inferSelect;
@@ -223,6 +227,7 @@ export const bookMetadataFetchQueue = pgTable(
(t) => [
index('bmfq_status_idx').on(t.status),
index('bmfq_created_at_idx').on(t.createdAt),
index('bmfq_status_created_book_idx').on(t.status, t.createdAt, t.bookId),
check('book_metadata_fetch_queue_status_chk', sql`${t.status} in ('queued', 'processing', 'failed')`),
check('book_metadata_fetch_queue_reason_chk', sql`${t.reason} in ('event_import', 'manual_trigger', 'manual_retry')`),
check('book_metadata_fetch_queue_attempt_count_nonnegative_chk', sql`${t.attemptCount} >= 0`),
server/src/db/schema/reader.ts
+1
View File
@@ -66,6 +66,7 @@ export const readingProgress = pgTable(
(t) => [
primaryKey({ columns: [t.bookFileId, t.userId] }),
index('reading_progress_user_id_idx').on(t.userId),
index('reading_progress_user_updated_at_idx').on(t.userId, t.updatedAt),
check('reading_progress_percentage_range_chk', sql`${t.percentage} >= 0 and ${t.percentage} <= 100`),
check('reading_progress_page_number_nonnegative_chk', sql`${t.pageNumber} is null or ${t.pageNumber} >= 0`),
check('reading_progress_position_seconds_nonnegative_chk', sql`${t.positionSeconds} is null or ${t.positionSeconds} >= 0`),
server/src/modules/authors/authors.service.ts
+9
View File
@@ -15,6 +15,7 @@ import type {
MergeAuthorsResult,
} from '@projectx/types';
import { assembleBookCards } from '../book/utils/assemble-book-cards';
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../common/constants/pagination.constants';
import type { RequestUser } from '../../common/types/request-user';
import { books } from '../../db/schema';
import { BookReadService } from '../book/book-read.service';
@@ -53,7 +54,14 @@ export class AuthorsService {
private readonly enrichmentOrchestrator: AuthorEnrichmentOrchestratorService,
) {}
private assertPaginationWindow(page: number, size: number): void {
if (!isOffsetWithinLimit(page * size)) {
throw new BadRequestException(`pagination window is too deep; page * size must be <= ${MAX_OFFSET_ROWS}`);
}
}
async findAll(user: RequestUser, dto: ListAuthorsDto): Promise<AuthorsPage> {
this.assertPaginationWindow(dto.page ?? 0, dto.size ?? 50);
const libraryIds = await this.resolveLibraryIds(user, dto.libraryId);
if (libraryIds.length === 0) {
return { items: [], total: 0, page: dto.page ?? 0, size: dto.size ?? 50 };
@@ -85,6 +93,7 @@ export class AuthorsService {
}
async findBooks(user: RequestUser, authorId: number, dto: ListAuthorBooksDto): Promise<BooksPage> {
this.assertPaginationWindow(dto.page ?? 0, dto.size ?? 50);
const libraryIds = await this.resolveLibraryIds(user, dto.libraryId);
if (libraryIds.length === 0) {
return { items: [], total: 0, page: dto.page ?? 0, size: dto.size ?? 50 };
server/src/modules/book-bucket/book-bucket-finalize.service.ts
+152 -23
View File
@@ -76,41 +76,66 @@ export class BookBucketFinalizeService implements OnModuleInit {
status?: string,
search?: string,
): Promise<BookBucketFinalizeResult> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : dedupeIds(fileIds ?? []);
const ids = selectAll ? [] : dedupeIds(fileIds ?? []);
const overrideMap = new Map((overrides ?? []).map((o) => [o.fileId, o]));
const results: BookBucketFinalizeFileResult[] = [];
let succeeded = 0;
let failed = 0;
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
const batch = ids.slice(i, i + BATCH_SIZE);
const rows = await this.repo.findByIds(batch);
const rowById = new Map(rows.map((row) => [row.id, row]));
if (selectAll) {
let afterId: number | undefined;
while (true) {
const rows = await this.repo.findSelectionBatch({
limit: BATCH_SIZE,
afterId,
excludedIds,
status,
search,
});
if (rows.length === 0) break;
for (const fileId of batch) {
const row = rowById.get(fileId);
if (!row) {
failed++;
results.push({
fileId,
fileName: `book-bucket-file-${fileId}`,
success: false,
message: 'Book Bucket file not found',
});
continue;
const duplicateLookup = await this.buildDuplicateLookup(rows, defaultLibraryId, overrideMap);
for (const row of rows) {
const result = await this.finalizeFile(row, defaultLibraryId, defaultFolderId, overrideMap, userId, isSuperuser, duplicateLookup);
results.push(result);
if (result.success) succeeded++;
else failed++;
}
const result = await this.finalizeFile(row, defaultLibraryId, defaultFolderId, overrideMap, userId, isSuperuser);
results.push(result);
if (result.success) succeeded++;
else failed++;
afterId = rows[rows.length - 1]?.id;
}
} else {
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
const batch = ids.slice(i, i + BATCH_SIZE);
const rows = await this.repo.findByIds(batch);
const rowById = new Map(rows.map((row) => [row.id, row]));
const duplicateLookup = await this.buildDuplicateLookup(rows, defaultLibraryId, overrideMap);
for (const fileId of batch) {
const row = rowById.get(fileId);
if (!row) {
failed++;
results.push({
fileId,
fileName: `book-bucket-file-${fileId}`,
success: false,
message: 'Book Bucket file not found',
});
continue;
}
const result = await this.finalizeFile(row, defaultLibraryId, defaultFolderId, overrideMap, userId, isSuperuser, duplicateLookup);
results.push(result);
if (result.success) succeeded++;
else failed++;
}
}
}
await this.emitSummary();
return { total: ids.length, succeeded, failed, results };
return { total: results.length, succeeded, failed, results };
}
private async finalizeFile(
@@ -120,6 +145,7 @@ export class BookBucketFinalizeService implements OnModuleInit {
overrideMap: Map<number, { libraryId?: number; folderId?: number }>,
userId: number,
isSuperuser: boolean,
duplicateLookup?: Map<string, number>,
): Promise<BookBucketFinalizeFileResult> {
try {
const override = overrideMap.get(row.id);
@@ -152,7 +178,7 @@ export class BookBucketFinalizeService implements OnModuleInit {
}
const meta = normalizeFinalizeMetadata(row.selectedMetadata ?? row.embeddedMetadata ?? {});
const existingBookId = await this.findDuplicate(libraryId, meta);
const existingBookId = await this.findDuplicate(libraryId, meta, duplicateLookup);
if (existingBookId !== null) {
return {
fileId: row.id,
@@ -257,7 +283,110 @@ export class BookBucketFinalizeService implements OnModuleInit {
});
}
private async findDuplicate(libraryId: number, meta: Pick<NormalizedFinalizeMetadata, 'isbn13' | 'isbn10' | 'title'>): Promise<number | null> {
private async buildDuplicateLookup(
rows: BookBucketFileRow[],
defaultLibraryId: number | undefined,
overrideMap: Map<number, { libraryId?: number; folderId?: number }>,
): Promise<Map<string, number>> {
const needsByLibrary = new Map<number, { isbn13: Set<string>; isbn10: Set<string>; titles: Set<string> }>();
for (const row of rows) {
const override = overrideMap.get(row.id);
const libraryId = override?.libraryId ?? row.targetLibraryId ?? defaultLibraryId ?? null;
if (libraryId === null) continue;
const meta = normalizeFinalizeMetadata(row.selectedMetadata ?? row.embeddedMetadata ?? {});
let bucket = needsByLibrary.get(libraryId);
if (!bucket) {
bucket = { isbn13: new Set(), isbn10: new Set(), titles: new Set() };
needsByLibrary.set(libraryId, bucket);
}
if (meta.isbn13) {
bucket.isbn13.add(meta.isbn13);
} else if (meta.isbn10) {
bucket.isbn10.add(meta.isbn10);
} else if (meta.title) {
bucket.titles.add(meta.title.toLowerCase());
}
}
const lookup = new Map<string, number>();
for (const [libraryId, values] of needsByLibrary) {
if (values.isbn13.size > 0) {
const rowsByIsbn13 = await this.db
.select({ bookId: bookMetadata.bookId, isbn13: bookMetadata.isbn13 })
.from(bookMetadata)
.innerJoin(books, eq(books.id, bookMetadata.bookId))
.where(and(eq(books.libraryId, libraryId), inArray(bookMetadata.isbn13, [...values.isbn13])));
for (const row of rowsByIsbn13) {
if (!row.isbn13) continue;
const key = this.buildDuplicateLookupKey(libraryId, { isbn13: row.isbn13, isbn10: null, title: null });
if (key && !lookup.has(key)) {
lookup.set(key, row.bookId);
}
}
}
if (values.isbn10.size > 0) {
const rowsByIsbn10 = await this.db
.select({ bookId: bookMetadata.bookId, isbn10: bookMetadata.isbn10 })
.from(bookMetadata)
.innerJoin(books, eq(books.id, bookMetadata.bookId))
.where(and(eq(books.libraryId, libraryId), inArray(bookMetadata.isbn10, [...values.isbn10])));
for (const row of rowsByIsbn10) {
if (!row.isbn10) continue;
const key = this.buildDuplicateLookupKey(libraryId, { isbn13: null, isbn10: row.isbn10, title: null });
if (key && !lookup.has(key)) {
lookup.set(key, row.bookId);
}
}
}
if (values.titles.size > 0) {
const rowsByTitle = await this.db
.select({
bookId: bookMetadata.bookId,
normalizedTitle: sql<string>`lower(${bookMetadata.title})`,
})
.from(bookMetadata)
.innerJoin(books, eq(books.id, bookMetadata.bookId))
.where(and(eq(books.libraryId, libraryId), inArray(sql<string>`lower(${bookMetadata.title})`, [...values.titles])));
for (const row of rowsByTitle) {
if (!row.normalizedTitle) continue;
const key = this.buildDuplicateLookupKey(libraryId, { isbn13: null, isbn10: null, title: row.normalizedTitle });
if (key && !lookup.has(key)) {
lookup.set(key, row.bookId);
}
}
}
}
return lookup;
}
private buildDuplicateLookupKey(libraryId: number, meta: Pick<NormalizedFinalizeMetadata, 'isbn13' | 'isbn10' | 'title'>): string | null {
const isbn = meta.isbn13 ?? meta.isbn10;
if (isbn) {
const isbnKind = meta.isbn13 ? 'isbn13' : 'isbn10';
return `library:${libraryId}|${isbnKind}:${isbn}`;
}
if (meta.title) {
return `library:${libraryId}|title:${meta.title.toLowerCase()}`;
}
return null;
}
private async findDuplicate(
libraryId: number,
meta: Pick<NormalizedFinalizeMetadata, 'isbn13' | 'isbn10' | 'title'>,
duplicateLookup?: Map<string, number>,
): Promise<number | null> {
const lookupKey = this.buildDuplicateLookupKey(libraryId, meta);
if (lookupKey && duplicateLookup?.has(lookupKey)) {
return duplicateLookup.get(lookupKey) ?? null;
}
const isbn = meta.isbn13 ?? meta.isbn10;
if (isbn) {
server/src/modules/book-bucket/book-bucket.repository.ts
+27 -1
View File
@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import { and, count, desc, asc, eq, ilike, inArray, notInArray, sum, type SQL } from 'drizzle-orm';
import { and, asc, count, desc, eq, gt, ilike, inArray, notInArray, sum, type SQL } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db';
@@ -25,6 +25,14 @@ export interface ListOptions {
search?: string;
}
export interface SelectionBatchOptions {
limit: number;
afterId?: number;
excludedIds?: number[];
status?: string;
search?: string;
}
@Injectable()
export class BookBucketRepository {
constructor(@Inject(DB) private readonly db: Db) {}
@@ -98,6 +106,24 @@ export class BookBucketRepository {
return this.db.select().from(bookBucketFiles).where(inArray(bookBucketFiles.id, ids));
}
async findSelectionBatch(options: SelectionBatchOptions): Promise<BookBucketFileRow[]> {
const conditions = this.buildSelectionConditions(options.status, options.search);
if (options.excludedIds?.length) conditions.push(notInArray(bookBucketFiles.id, options.excludedIds));
if (options.afterId !== undefined) conditions.push(gt(bookBucketFiles.id, options.afterId));
const where = conditions.length ? and(...conditions) : undefined;
return this.db.select().from(bookBucketFiles).where(where).orderBy(asc(bookBucketFiles.id)).limit(options.limit);
}
async setTargetsByIds(ids: number[], targetLibraryId: number | null, targetFolderId: number | null): Promise<number> {
if (ids.length === 0) return 0;
const updated = await this.db
.update(bookBucketFiles)
.set({ targetLibraryId, targetFolderId })
.where(inArray(bookBucketFiles.id, ids))
.returning({ id: bookBucketFiles.id });
return updated.length;
}
async countsByStatus(): Promise<{ pending: number; ready: number; error: number; total: number }> {
const rows = await this.db
.select({
server/src/modules/book-bucket/book-bucket.service.ts
+183 -80
View File
@@ -12,6 +12,7 @@ import { BookBucketIngestService } from './book-bucket-ingest.service';
import type { BookBucketFileRow } from '../../db/schema';
type Db = NodePgDatabase<typeof schema>;
const BULK_SELECTION_BATCH_SIZE = 500;
@Injectable()
export class BookBucketService {
@@ -63,14 +64,21 @@ export class BookBucketService {
}
async bulkDiscard(fileIds: number[], selectAll?: boolean, excludedIds?: number[], status?: string, search?: string): Promise<void> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : fileIds;
if (!ids.length) return;
const rows = await this.repo.findByIds(ids);
for (const row of rows) {
await this.cleanupFiles(row);
}
await this.repo.deleteByIds(ids);
await this.processSelectionRows(
{
fileIds,
selectAll,
excludedIds,
status,
search,
},
async (rows) => {
for (const row of rows) {
await this.cleanupFiles(row);
}
await this.repo.deleteByIds(rows.map((row) => row.id));
},
);
}
async bulkEdit(
@@ -83,35 +91,43 @@ export class BookBucketService {
status?: string,
search?: string,
): Promise<{ total: number; updated: number; failed: number }> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : (fileIds ?? []);
const rows = await this.repo.findByIds(ids);
let updated = 0;
let failed = 0;
const total = await this.processSelectionRows(
{
fileIds: fileIds ?? [],
selectAll,
excludedIds,
status,
search,
},
async (rows) => {
for (const row of rows) {
try {
const current: Record<string, unknown> = { ...(row.selectedMetadata ?? row.embeddedMetadata ?? {}) };
for (const row of rows) {
try {
const current: Record<string, unknown> = { ...(row.selectedMetadata ?? row.embeddedMetadata ?? {}) };
for (const field of enabledFields) {
const value = (fields as Record<string, unknown>)[field];
if (value === undefined) continue;
for (const field of enabledFields) {
const value = (fields as Record<string, unknown>)[field];
if (value === undefined) continue;
if (mergeArrays && Array.isArray(value) && Array.isArray(current[field])) {
const merged = [...new Set([...(current[field] as string[]), ...(value as string[])])];
current[field] = merged;
} else {
current[field] = value;
}
}
if (mergeArrays && Array.isArray(value) && Array.isArray(current[field])) {
const merged = [...new Set([...(current[field] as string[]), ...(value as string[])])];
current[field] = merged;
} else {
current[field] = value;
await this.repo.update(row.id, { selectedMetadata: current as BookBucketMetadata });
updated++;
} catch {
failed++;
}
}
},
);
await this.repo.update(row.id, { selectedMetadata: current as BookBucketMetadata });
updated++;
} catch {
failed++;
}
}
return { total: rows.length, updated, failed };
return { total, updated, failed };
}
async bulkApplyFetched(
@@ -121,28 +137,34 @@ export class BookBucketService {
status?: string,
search?: string,
): Promise<{ total: number; applied: number; skipped: number; skippedEdited: number }> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : fileIds;
if (!ids.length) return { total: 0, applied: 0, skipped: 0, skippedEdited: 0 };
const rows = await this.repo.findByIds(ids);
let applied = 0;
let skipped = 0;
let skippedEdited = 0;
const total = await this.processSelectionRows(
{
fileIds,
selectAll,
excludedIds,
status,
search,
},
async (rows) => {
for (const row of rows) {
if (row.metadataEditedAt) {
skippedEdited++;
continue;
}
if (!row.fetchedMetadata) {
skipped++;
continue;
}
await this.repo.update(row.id, { selectedMetadata: row.fetchedMetadata, metadataEditedAt: null });
applied++;
}
},
);
for (const row of rows) {
if (row.metadataEditedAt) {
skippedEdited++;
continue;
}
if (!row.fetchedMetadata) {
skipped++;
continue;
}
await this.repo.update(row.id, { selectedMetadata: row.fetchedMetadata, metadataEditedAt: null });
applied++;
}
return { total: rows.length, applied, skipped, skippedEdited };
return { total, applied, skipped, skippedEdited };
}
async bulkRetryFetch(
@@ -152,16 +174,25 @@ export class BookBucketService {
status?: string,
search?: string,
): Promise<{ total: number; queued: number }> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : (fileIds ?? []);
if (!ids.length) return { total: 0, queued: 0 };
let queued = 0;
const total = await this.processSelectionRows(
{
fileIds: fileIds ?? [],
selectAll,
excludedIds,
status,
search,
},
(rows) => {
const errorRows = rows.filter((row) => row.status === 'error');
queued += errorRows.length;
for (const row of errorRows) {
void this.ingestService.retryFetch(row.id);
}
},
);
const rows = await this.repo.findByIds(ids);
const errorRows = rows.filter((r) => r.status === 'error');
for (const row of errorRows) {
void this.ingestService.retryFetch(row.id);
}
return { total: rows.length, queued: errorRows.length };
return { total, queued };
}
async bulkSetTarget(
@@ -174,24 +205,25 @@ export class BookBucketService {
search?: string,
): Promise<{ total: number; updated: number; failed: number }> {
await this.assertValidTarget(targetLibraryId, targetFolderId);
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : fileIds;
if (!ids.length) return { total: 0, updated: 0, failed: 0 };
const rows = await this.repo.findByIds(ids);
let updated = 0;
let failed = 0;
for (const row of rows) {
try {
await this.repo.update(row.id, { targetLibraryId: targetLibraryId ?? null, targetFolderId: targetFolderId ?? null });
updated++;
} catch {
failed++;
}
}
return { total: rows.length, updated, failed };
const total = await this.processSelectionRows(
{
fileIds,
selectAll,
excludedIds,
status,
search,
},
async (rows) => {
updated += await this.repo.setTargetsByIds(
rows.map((row) => row.id),
targetLibraryId ?? null,
targetFolderId ?? null,
);
},
);
const failed = total - updated;
return { total, updated, failed };
}
async selectionSummary(
@@ -201,12 +233,28 @@ export class BookBucketService {
status?: string,
search?: string,
): Promise<{ total: number; withDestination: number; withoutDestination: number }> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : fileIds;
if (!ids.length) return { total: 0, withDestination: 0, withoutDestination: 0 };
const destinationPairCounts = new Map<string, number>();
const folderIdSet = new Set<number>();
const total = await this.processSelectionRows(
{
fileIds,
selectAll,
excludedIds,
status,
search,
},
(rows) => {
for (const row of rows) {
if (row.targetLibraryId === null || row.targetFolderId === null) continue;
folderIdSet.add(row.targetFolderId);
const key = `${row.targetFolderId}:${row.targetLibraryId}`;
destinationPairCounts.set(key, (destinationPairCounts.get(key) ?? 0) + 1);
}
},
);
if (total === 0) return { total: 0, withDestination: 0, withoutDestination: 0 };
const rows = await this.repo.findByIds(ids);
const candidates = rows.filter((row) => row.targetLibraryId !== null && row.targetFolderId !== null);
const folderIds = [...new Set(candidates.map((row) => row.targetFolderId!).filter((id): id is number => id != null))];
const folderIds = [...folderIdSet];
const folderRows = folderIds.length
? await this.db
.select({ id: libraryFolders.id, libraryId: libraryFolders.libraryId })
@@ -214,8 +262,17 @@ export class BookBucketService {
.where(inArray(libraryFolders.id, folderIds))
: [];
const folderById = new Map(folderRows.map((row) => [row.id, row.libraryId]));
const withDestination = candidates.filter((row) => folderById.get(row.targetFolderId!) === row.targetLibraryId).length;
return { total: rows.length, withDestination, withoutDestination: rows.length - withDestination };
let withDestination = 0;
for (const [key, count] of destinationPairCounts) {
const [folderIdRaw, libraryIdRaw] = key.split(':');
const folderId = Number(folderIdRaw);
const libraryId = Number(libraryIdRaw);
if (folderById.get(folderId) === libraryId) {
withDestination += count;
}
}
return { total, withDestination, withoutDestination: total - withDestination };
}
async getSummary(): Promise<BookBucketSummary> {
@@ -265,6 +322,48 @@ export class BookBucketService {
throw new BadRequestException('Destination folder does not belong to destination library');
}
}
private async processSelectionRows(
options: {
fileIds: number[];
selectAll?: boolean;
excludedIds?: number[];
status?: string;
search?: string;
},
processBatch: (rows: BookBucketFileRow[]) => Promise<void> | void,
): Promise<number> {
let total = 0;
if (options.selectAll) {
let afterId: number | undefined;
while (true) {
const rows = await this.repo.findSelectionBatch({
limit: BULK_SELECTION_BATCH_SIZE,
afterId,
excludedIds: options.excludedIds,
status: options.status,
search: options.search,
});
if (rows.length === 0) break;
await processBatch(rows);
total += rows.length;
afterId = rows[rows.length - 1]?.id;
}
return total;
}
const ids = dedupeIds(options.fileIds);
for (let index = 0; index < ids.length; index += BULK_SELECTION_BATCH_SIZE) {
const batchIds = ids.slice(index, index + BULK_SELECTION_BATCH_SIZE);
const rows = await this.repo.findByIds(batchIds);
if (rows.length === 0) continue;
await processBatch(rows);
total += rows.length;
}
return total;
}
}
function toDto(row: BookBucketFileRow): BookBucketFile {
@@ -295,3 +394,7 @@ async function safeUnlink(path: string): Promise<void> {
// file may already be deleted
}
}
function dedupeIds(ids: number[]): number[] {
return [...new Set(ids.filter((id): id is number => Number.isInteger(id) && id > 0))];
}
server/src/modules/book-metadata-fetch/book-metadata-fetch-orchestrator.service.ts
+5 -8
View File
@@ -17,6 +17,7 @@ import { BookMetadataFetchSessionService } from './book-metadata-fetch-session.s
const POLL_INTERVAL_MS = 4_000;
const BATCH_SIZE = 1;
const SCHEDULE_BATCH_SIZE = 1000;
@Injectable()
export class BookMetadataFetchOrchestratorService implements OnApplicationBootstrap, OnModuleDestroy {
@@ -73,10 +74,8 @@ export class BookMetadataFetchOrchestratorService implements OnApplicationBootst
async triggerGlobal(): Promise<number> {
const config = await this.configService.getGlobalConfig();
const bookIds = await this.queueRepo.fetchEligibleBookIds(config);
if (bookIds.length === 0) return 0;
const queued = await this.queueRepo.upsertSchedule(bookIds, 'manual_trigger');
const queued = await this.queueRepo.scheduleEligibleBooksInBatches(config, 'manual_trigger', undefined, SCHEDULE_BATCH_SIZE);
if (queued === 0) return 0;
if (queued > 0) {
this.session.addToTotal(queued);
await this.unpauseIfNeeded();
@@ -90,10 +89,8 @@ export class BookMetadataFetchOrchestratorService implements OnApplicationBootst
const config = await this.configService.getEffectiveConfig(libraryId);
if (!config.enabled) return 0;
const bookIds = await this.queueRepo.fetchEligibleBookIds(config, libraryId);
if (bookIds.length === 0) return 0;
const queued = await this.queueRepo.upsertSchedule(bookIds, 'manual_trigger');
const queued = await this.queueRepo.scheduleEligibleBooksInBatches(config, 'manual_trigger', libraryId, SCHEDULE_BATCH_SIZE);
if (queued === 0) return 0;
if (queued > 0) {
this.session.addToTotal(queued);
await this.unpauseIfNeeded();
server/src/modules/book-metadata-fetch/book-metadata-fetch-queue.repository.ts
+33
View File
@@ -178,6 +178,39 @@ export class BookMetadataFetchQueueRepository {
return rows.map((r) => r.bookId);
}
async scheduleEligibleBooksInBatches(
config: BookMetadataFetchConfig,
reason: BookMetadataFetchReason,
libraryId?: number,
batchSize = 1000,
): Promise<number> {
const whereClause = this.buildEligibleBooksWhereClause(config, libraryId);
if (!whereClause || batchSize <= 0) return 0;
let cursorBookId = 0;
let totalQueued = 0;
for (;;) {
const rows = await this.db
.select({ bookId: bookMetadata.bookId })
.from(bookMetadata)
.innerJoin(books, eq(books.id, bookMetadata.bookId))
.where(and(whereClause, sql`${bookMetadata.bookId} > ${cursorBookId}`))
.orderBy(asc(bookMetadata.bookId))
.limit(batchSize);
if (rows.length === 0) break;
totalQueued += await this.upsertSchedule(
rows.map((row) => row.bookId),
reason,
);
cursorBookId = rows[rows.length - 1]!.bookId;
}
return totalQueued;
}
async countEligibleBooks(config: BookMetadataFetchConfig, libraryId?: number): Promise<number> {
const whereClause = this.buildEligibleBooksWhereClause(config, libraryId);
if (!whereClause) return 0;
server/src/modules/book/book-query-builder.service.test.ts
+44 -57
View File
@@ -329,8 +329,8 @@ describe('textRuleToSql (via title)', () => {
['endsWith', 'Dune', { type: 'ilike', pattern: '%Dune' }],
['eq', 'Dune', { type: 'ilike', pattern: 'Dune' }],
['notEq', 'Dune', { type: 'not' }],
['isEmpty', undefined, { type: 'isNull' }],
['isNotEmpty', undefined, { type: 'isNotNull' }],
['isEmpty', undefined, { type: 'or' }],
['isNotEmpty', undefined, { type: 'and' }],
] as const)('operator %s produces the correct SQL shape', (operator, value, expected) => {
const { builder } = makeBuilder();
const rule: Record<string, unknown> = { type: 'rule', field: 'title', operator };
@@ -424,23 +424,22 @@ describe('numericRuleToSql (via pageCount and publishedYear)', () => {
});
describe('authorRuleToSql', () => {
it('includesAny with values produces inArray(books.id, subquery)', () => {
it('includesAny with values produces an EXISTS subquery', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'author', operator: 'includesAny', value: ['Tolkien'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where).right).toMatchObject({ type: 'subquery' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
it('excludesAll with values produces not(inArray(books.id, subquery))', () => {
it('excludesAll with values produces not(EXISTS subquery)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'author', operator: 'excludesAll', value: ['Tolkien'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('excludesAll with empty array produces always-true branch', () => {
@@ -449,31 +448,28 @@ describe('authorRuleToSql', () => {
expect(getRuleSql(where)).toMatchObject({ type: 'sql', text: '1 = 1' });
});
it('isEmpty produces not(inArray(books.id, subquery with no where clause))', () => {
it('isEmpty produces not(EXISTS subquery with no where clause)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'author', operator: 'isEmpty' }) as never, USER_CTX) as any;
const clause = getRuleSql(where);
expect(clause).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(clause).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('isNotEmpty produces inArray(books.id, subquery with no where clause)', () => {
it('isNotEmpty produces EXISTS subquery with no where clause', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'author', operator: 'isNotEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
});
describe('genreRuleToSql', () => {
it('includesAny produces inArray with eq-based subquery (not ilike like author)', () => {
it('includesAny produces an EXISTS subquery with eq filters', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'genre', operator: 'includesAny', value: ['Fiction'] }) as never,
USER_CTX,
) as any;
const clause = getRuleSql(where);
expect(clause).toMatchObject({ type: 'inArray' });
expect(clause.right.whereClause).toMatchObject({ type: 'or' });
expect(clause.right.whereClause.clauses[0]).toMatchObject({ type: 'eq' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
it('includesAll produces one subquery per value', () => {
@@ -482,13 +478,13 @@ describe('genreRuleToSql', () => {
expect(db.select).toHaveBeenCalledTimes(2);
});
it('excludesAll with values produces not(inArray(books.id, subquery))', () => {
it('excludesAll with values produces not(EXISTS subquery)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'genre', operator: 'excludesAll', value: ['Fiction'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('excludesAll with empty array produces always-true branch', () => {
@@ -497,27 +493,27 @@ describe('genreRuleToSql', () => {
expect(getRuleSql(where)).toMatchObject({ type: 'sql', text: '1 = 1' });
});
it('isEmpty produces not(inArray(books.id, subquery))', () => {
it('isEmpty produces not(EXISTS subquery)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'genre', operator: 'isEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('isNotEmpty produces inArray(books.id, subquery)', () => {
it('isNotEmpty produces EXISTS subquery', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'genre', operator: 'isNotEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
});
describe('tagRuleToSql', () => {
it('includesAny produces inArray(books.id, subquery)', () => {
it('includesAny produces an EXISTS subquery', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'tag', operator: 'includesAny', value: ['favourite'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
it('includesAll produces one subquery per value', () => {
@@ -526,13 +522,13 @@ describe('tagRuleToSql', () => {
expect(db.select).toHaveBeenCalledTimes(2);
});
it('excludesAll with values produces not(inArray)', () => {
it('excludesAll with values produces not(EXISTS subquery)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'tag', operator: 'excludesAll', value: ['favourite'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('excludesAll with empty array produces always-true branch', () => {
@@ -541,36 +537,36 @@ describe('tagRuleToSql', () => {
expect(getRuleSql(where)).toMatchObject({ type: 'sql', text: '1 = 1' });
});
it('isEmpty produces not(inArray)', () => {
it('isEmpty produces not(EXISTS subquery)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'tag', operator: 'isEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('isNotEmpty produces inArray', () => {
it('isNotEmpty produces EXISTS subquery', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'tag', operator: 'isNotEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
});
describe('collectionRuleToSql', () => {
it('includesAny produces inArray(books.id, user-scoped subquery)', () => {
it('includesAny produces user-scoped EXISTS subquery', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'collection', operator: 'includesAny', value: ['Reading'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
it('excludesAll with values produces not(inArray)', () => {
it('excludesAll with values produces not(EXISTS)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'collection', operator: 'excludesAll', value: ['Reading'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('excludesAll with empty array produces always-true branch', () => {
@@ -579,27 +575,27 @@ describe('collectionRuleToSql', () => {
expect(getRuleSql(where)).toMatchObject({ type: 'sql', text: '1 = 1' });
});
it('isEmpty produces not(inArray)', () => {
it('isEmpty produces not(EXISTS)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'collection', operator: 'isEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('isNotEmpty produces inArray', () => {
it('isNotEmpty produces EXISTS', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'collection', operator: 'isNotEmpty' }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
});
describe('formatRuleToSql', () => {
it('includesAny produces inArray(books.id, subquery with inArray on format column)', () => {
it('includesAny produces EXISTS subquery with format filter', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(
wrapRule({ type: 'rule', field: 'format', operator: 'includesAny', value: ['PDF', 'EPUB'] }) as never,
USER_CTX,
) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'inArray' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
it('includesAny with empty array produces always-false branch', () => {
@@ -608,10 +604,10 @@ describe('formatRuleToSql', () => {
expect(getRuleSql(where)).toMatchObject({ type: 'sql', text: '1 = 0' });
});
it('excludesAll with values produces not(inArray)', () => {
it('excludesAll with values produces not(EXISTS)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'format', operator: 'excludesAll', value: ['PDF'] }) as never, USER_CTX) as any;
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(getRuleSql(where)).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('excludesAll with empty array produces always-true branch', () => {
@@ -685,32 +681,23 @@ describe('statusRuleToSql (fileAvailability)', () => {
});
describe('readProgressRuleToSql', () => {
it('isUnread produces not(inArray(books.id, subquery))', () => {
it('isUnread produces not(EXISTS subquery)', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'readProgress', operator: 'isUnread' }) as never, USER_CTX) as any;
const clause = getRuleSql(where);
expect(clause).toMatchObject({ type: 'not', value: { type: 'inArray' } });
expect(clause).toMatchObject({ type: 'not', value: { type: 'sql' } });
});
it('isInProgress produces inArray(books.id, subquery with percentage between 0 and 100)', () => {
it('isInProgress produces EXISTS subquery with percentage between 0 and 100', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'readProgress', operator: 'isInProgress' }) as never, USER_CTX) as any;
const clause = getRuleSql(where);
expect(clause).toMatchObject({ type: 'inArray' });
expect(clause.right).toMatchObject({ type: 'subquery' });
expect(clause.right.whereClause).toMatchObject({ type: 'and' });
expect(clause.right.whereClause.clauses).toHaveLength(3);
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
it('isFinished produces inArray(books.id, subquery with percentage >= 100)', () => {
it('isFinished produces EXISTS subquery with percentage >= 100', () => {
const { builder } = makeBuilder();
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'readProgress', operator: 'isFinished' }) as never, USER_CTX) as any;
const clause = getRuleSql(where);
expect(clause).toMatchObject({ type: 'inArray' });
expect(clause.right).toMatchObject({ type: 'subquery' });
expect(clause.right.whereClause).toMatchObject({ type: 'and' });
expect(clause.right.whereClause.clauses).toHaveLength(2);
expect(clause.right.whereClause.clauses[1]).toMatchObject({ type: 'gte' });
expect(getRuleSql(where)).toMatchObject({ type: 'sql' });
});
});
server/src/modules/book/book-query-builder.service.ts
+86 -46
View File
@@ -76,12 +76,12 @@ export class BookQueryBuilder {
} else if (field === 'readProgress') {
if (userId === undefined) throw new BadRequestException('readProgress sort requires an authenticated user');
result.push(
sql`(SELECT rp.percentage FROM reading_progress rp INNER JOIN book_files bf ON rp.book_file_id = bf.id WHERE bf.book_id = books.id AND rp.user_id = ${userId} ORDER BY rp.percentage DESC LIMIT 1) ${sql.raw(D)} NULLS LAST`,
sql`(SELECT max(rp.percentage) FROM reading_progress rp INNER JOIN book_files bf ON rp.book_file_id = bf.id WHERE bf.book_id = books.id AND rp.user_id = ${userId}) ${sql.raw(D)} NULLS LAST`,
);
} else if (field === 'lastReadAt') {
if (userId === undefined) throw new BadRequestException('lastReadAt sort requires an authenticated user');
result.push(
sql`(SELECT rp.updated_at FROM reading_progress rp INNER JOIN book_files bf ON rp.book_file_id = bf.id WHERE bf.book_id = books.id AND rp.user_id = ${userId} ORDER BY rp.updated_at DESC LIMIT 1) ${sql.raw(D)} NULLS LAST`,
sql`(SELECT max(rp.updated_at) FROM reading_progress rp INNER JOIN book_files bf ON rp.book_file_id = bf.id WHERE bf.book_id = books.id AND rp.user_id = ${userId}) ${sql.raw(D)} NULLS LAST`,
);
} else if (field === 'finishedAt') {
if (userId === undefined) throw new BadRequestException('finishedAt sort requires an authenticated user');
@@ -89,7 +89,10 @@ export class BookQueryBuilder {
sql`(SELECT ubs.finished_at FROM user_book_status ubs WHERE ubs.book_id = books.id AND ubs.user_id = ${userId}) ${sql.raw(D)} NULLS LAST`,
);
} else if (field === 'random') {
result.push(sql.raw('RANDOM()'));
const daySeed = Math.floor(Date.now() / 86_400_000);
const scopedSeed = daySeed + (userId ?? 0);
result.push(sql`md5(${books.id}::text || ':' || ${scopedSeed}::text) ${sql.raw(D)}`);
result.push(sql`${books.id} ${sql.raw(D)}`);
} else {
const col = SORT_FIELD_MAP[field];
if (!col) continue;
@@ -185,9 +188,9 @@ export class BookQueryBuilder {
case 'notEq':
return not(ilike(col, value!));
case 'isEmpty':
return isNull(col);
return or(isNull(col), eq(col, ''))!;
case 'isNotEmpty':
return isNotNull(col);
return and(isNotNull(col), ne(col, ''))!;
default:
throw new BadRequestException(`Invalid operator '${operator}' for text field`);
}
@@ -240,61 +243,85 @@ export class BookQueryBuilder {
}
private authorRuleToSql(operator: string, values?: string[]): SQL {
const sq = (whereClause?: SQL) =>
this.db.select({ bookId: bookAuthors.bookId }).from(bookAuthors).innerJoin(authors, eq(bookAuthors.authorId, authors.id)).where(whereClause);
const existsAuthor = (whereClause?: SQL) => {
const predicates: SQL[] = [eq(bookAuthors.bookId, books.id)];
if (whereClause) predicates.push(whereClause);
const sq = this.db
.select({ one: sql`1` })
.from(bookAuthors)
.innerJoin(authors, eq(bookAuthors.authorId, authors.id))
.where(and(...predicates)!);
return sql`exists (${sq})`;
};
switch (operator) {
case 'includesAny':
if (!values?.length) return sql`1 = 0`;
return inArray(books.id, sq(or(...values.map((v) => ilike(authors.name, `%${v}%`)))));
return existsAuthor(or(...values.map((v) => ilike(authors.name, `%${v}%`)))!);
case 'includesAll':
if (!values?.length) return sql`1 = 0`;
return and(...values.map((v) => inArray(books.id, sq(ilike(authors.name, `%${v}%`)))))!;
return and(...values.map((v) => existsAuthor(ilike(authors.name, `%${v}%`))))!;
case 'excludesAll':
if (!values?.length) return sql`1 = 1`;
return not(inArray(books.id, sq(or(...values.map((v) => ilike(authors.name, `%${v}%`))))));
return not(existsAuthor(or(...values.map((v) => ilike(authors.name, `%${v}%`)))!));
case 'isEmpty':
return not(inArray(books.id, sq()));
return not(existsAuthor());
case 'isNotEmpty':
return inArray(books.id, sq());
return existsAuthor();
default:
throw new BadRequestException(`Invalid operator '${operator}' for author field`);
}
}
private genreRuleToSql(operator: string, values?: string[]): SQL {
const sq = (whereClause?: SQL) =>
this.db.select({ bookId: bookGenres.bookId }).from(bookGenres).innerJoin(genres, eq(bookGenres.genreId, genres.id)).where(whereClause);
const existsGenre = (whereClause?: SQL) => {
const predicates: SQL[] = [eq(bookGenres.bookId, books.id)];
if (whereClause) predicates.push(whereClause);
const sq = this.db
.select({ one: sql`1` })
.from(bookGenres)
.innerJoin(genres, eq(bookGenres.genreId, genres.id))
.where(and(...predicates)!);
return sql`exists (${sq})`;
};
switch (operator) {
case 'includesAny':
if (!values?.length) return sql`1 = 0`;
return inArray(books.id, sq(or(...values.map((v) => eq(genres.name, v)))));
return existsGenre(or(...values.map((v) => eq(genres.name, v)))!);
case 'includesAll':
if (!values?.length) return sql`1 = 0`;
return and(...values.map((v) => inArray(books.id, sq(eq(genres.name, v)))))!;
return and(...values.map((v) => existsGenre(eq(genres.name, v))))!;
case 'excludesAll':
if (!values?.length) return sql`1 = 1`;
return not(inArray(books.id, sq(or(...values.map((v) => eq(genres.name, v))))));
return not(existsGenre(or(...values.map((v) => eq(genres.name, v)))!));
case 'isEmpty':
return not(inArray(books.id, sq()));
return not(existsGenre());
case 'isNotEmpty':
return inArray(books.id, sq());
return existsGenre();
default:
throw new BadRequestException(`Invalid operator '${operator}' for genre field`);
}
}
private formatRuleToSql(operator: string, values?: string[]): SQL {
const sq = (whereClause?: SQL) => this.db.select({ bookId: bookFiles.bookId }).from(bookFiles).where(whereClause);
const existsFormat = (whereClause?: SQL) => {
const predicates: SQL[] = [eq(bookFiles.bookId, books.id)];
if (whereClause) predicates.push(whereClause);
const sq = this.db
.select({ one: sql`1` })
.from(bookFiles)
.where(and(...predicates)!);
return sql`exists (${sq})`;
};
switch (operator) {
case 'includesAny':
if (!values?.length) return sql`1 = 0`;
return inArray(books.id, sq(inArray(bookFiles.format, values)));
return existsFormat(inArray(bookFiles.format, values));
case 'excludesAll':
if (!values?.length) return sql`1 = 1`;
return not(inArray(books.id, sq(inArray(bookFiles.format, values))));
return not(existsFormat(inArray(bookFiles.format, values)));
default:
throw new BadRequestException(`Invalid operator '${operator}' for format field`);
}
@@ -349,46 +376,55 @@ export class BookQueryBuilder {
private readProgressRuleToSql(operator: string, userId?: number): SQL {
if (userId === undefined) throw new BadRequestException('Reading progress filter requires an authenticated user');
const sq = (whereClause: SQL) =>
this.db
.select({ bookId: bookFiles.bookId })
const existsReadingProgress = (whereClause: SQL) => {
const sq = this.db
.select({ one: sql`1` })
.from(readingProgress)
.innerJoin(bookFiles, eq(readingProgress.bookFileId, bookFiles.id))
.where(whereClause);
.where(and(eq(bookFiles.bookId, books.id), whereClause)!);
return sql`exists (${sq})`;
};
switch (operator) {
case 'isUnread':
return not(inArray(books.id, sq(and(eq(readingProgress.userId, userId), gt(readingProgress.percentage, 0))!)));
return not(existsReadingProgress(and(eq(readingProgress.userId, userId), gt(readingProgress.percentage, 0))!));
case 'isInProgress':
return inArray(
books.id,
sq(and(eq(readingProgress.userId, userId), gt(readingProgress.percentage, 0), lt(readingProgress.percentage, 100))!),
return existsReadingProgress(
and(eq(readingProgress.userId, userId), gt(readingProgress.percentage, 0), lt(readingProgress.percentage, 100))!,
);
case 'isFinished':
return inArray(books.id, sq(and(eq(readingProgress.userId, userId), gte(readingProgress.percentage, 100))!));
return existsReadingProgress(and(eq(readingProgress.userId, userId), gte(readingProgress.percentage, 100))!);
default:
throw new BadRequestException(`Invalid operator '${operator}' for readProgress field`);
}
}
private tagRuleToSql(operator: string, values?: string[]): SQL {
const sq = (whereClause?: SQL) =>
this.db.select({ bookId: bookTags.bookId }).from(bookTags).innerJoin(tags, eq(bookTags.tagId, tags.id)).where(whereClause);
const existsTag = (whereClause?: SQL) => {
const predicates: SQL[] = [eq(bookTags.bookId, books.id)];
if (whereClause) predicates.push(whereClause);
const sq = this.db
.select({ one: sql`1` })
.from(bookTags)
.innerJoin(tags, eq(bookTags.tagId, tags.id))
.where(and(...predicates)!);
return sql`exists (${sq})`;
};
switch (operator) {
case 'includesAny':
if (!values?.length) return sql`1 = 0`;
return inArray(books.id, sq(or(...values.map((v) => eq(tags.name, v)))));
return existsTag(or(...values.map((v) => eq(tags.name, v)))!);
case 'includesAll':
if (!values?.length) return sql`1 = 0`;
return and(...values.map((v) => inArray(books.id, sq(eq(tags.name, v)))))!;
return and(...values.map((v) => existsTag(eq(tags.name, v))))!;
case 'excludesAll':
if (!values?.length) return sql`1 = 1`;
return not(inArray(books.id, sq(or(...values.map((v) => eq(tags.name, v))))));
return not(existsTag(or(...values.map((v) => eq(tags.name, v)))!));
case 'isEmpty':
return not(inArray(books.id, sq()));
return not(existsTag());
case 'isNotEmpty':
return inArray(books.id, sq());
return existsTag();
default:
throw new BadRequestException(`Invalid operator '${operator}' for tag field`);
}
@@ -396,24 +432,28 @@ export class BookQueryBuilder {
private collectionRuleToSql(operator: string, values?: string[], userId?: number): SQL {
if (userId === undefined) throw new BadRequestException('Collection filter requires an authenticated user');
const sq = (whereClause?: SQL) =>
this.db
.select({ bookId: collectionBooks.bookId })
const existsCollection = (whereClause?: SQL) => {
const predicates: SQL[] = [eq(collectionBooks.bookId, books.id)];
if (whereClause) predicates.push(whereClause);
const sq = this.db
.select({ one: sql`1` })
.from(collectionBooks)
.innerJoin(collections, and(eq(collectionBooks.collectionId, collections.id), eq(collections.userId, userId)))
.where(whereClause);
.where(and(...predicates)!);
return sql`exists (${sq})`;
};
switch (operator) {
case 'includesAny':
if (!values?.length) return sql`1 = 0`;
return inArray(books.id, sq(or(...values.map((v) => eq(collections.name, v)))));
return existsCollection(or(...values.map((v) => eq(collections.name, v)))!);
case 'excludesAll':
if (!values?.length) return sql`1 = 1`;
return not(inArray(books.id, sq(or(...values.map((v) => eq(collections.name, v))))));
return not(existsCollection(or(...values.map((v) => eq(collections.name, v)))!));
case 'isEmpty':
return not(inArray(books.id, sq()));
return not(existsCollection());
case 'isNotEmpty':
return inArray(books.id, sq());
return existsCollection();
default:
throw new BadRequestException(`Invalid operator '${operator}' for collection field`);
}
server/src/modules/book/book.service.ts
+9
View File
@@ -12,6 +12,7 @@ import { ConfigService } from '@nestjs/config';
import { access, readdir, rm, stat } from 'fs/promises';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../common/constants/pagination.constants';
import { extractEpubMetadata } from '../metadata/lib/epub';
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
import { parseFb2File } from '../metadata/lib/fb2-parser';
@@ -201,7 +202,14 @@ export class BookService {
};
}
private assertPaginationWindow(page: number, size: number): void {
if (!isOffsetWithinLimit(page * size)) {
throw new BadRequestException(`pagination window is too deep; page * size must be <= ${MAX_OFFSET_ROWS}`);
}
}
async queryForLibrary(user: RequestUser, libraryId: number, query: BookQuery): Promise<BooksPage> {
this.assertPaginationWindow(query.pagination.page, query.pagination.size);
await this.libraryService.verifyUserAccess(user.id, libraryId, this.isSuperuser(user));
const where = this.queryBuilder.buildWhere(query.filter, { accessibleLibraryIds: [libraryId], implicitLibraryId: libraryId, userId: user.id });
const orderBy = this.queryBuilder.buildOrderBy(query.sort, user.id);
@@ -221,6 +229,7 @@ export class BookService {
}
async globalQuery(user: RequestUser, query: BookQuery): Promise<BooksPage> {
this.assertPaginationWindow(query.pagination.page, query.pagination.size);
const libs = await this.libraryService.findAll(user);
const accessibleLibraryIds = libs.map((l) => l.id);
const where = this.queryBuilder.buildWhere(query.filter, { accessibleLibraryIds, userId: user.id });
server/src/modules/book/pipes/book-query.pipe.ts
+10
View File
@@ -3,6 +3,7 @@ import { z } from 'zod';
import type { BookQuery } from '@projectx/types';
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../../common/constants/pagination.constants';
import { groupRuleSchema } from '../utils/group-rule.validator';
const SORT_FIELDS = [
@@ -39,6 +40,15 @@ const bookQuerySchema = z.object({
page: z.number().int().min(0).default(0),
size: z.number().int().min(1).max(200).default(50),
})
.superRefine((pagination, ctx) => {
if (!isOffsetWithinLimit(pagination.page * pagination.size)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `pagination window is too deep; page * size must be <= ${MAX_OFFSET_ROWS}`,
path: ['page'],
});
}
})
.default({ page: 0, size: 50 }),
});
server/src/modules/collection/collection.controller.ts
+8 -2
View File
@@ -15,6 +15,7 @@ import {
} from '@nestjs/common';
import { AuditAction, AuditResource } from '@projectx/types';
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../common/constants/pagination.constants';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
import { Auditable } from '../../common/decorators/auditable.decorator';
import type { RequestUser } from '../../common/types/request-user';
@@ -26,6 +27,7 @@ import { CollectionService } from './collection.service';
const BOOK_IDS_QUERY_INVALID_MESSAGE = 'bookIds must be a comma-separated list of positive integers';
const PAGE_QUERY_INVALID_MESSAGE = 'page must be greater than or equal to 0';
const PAGE_WINDOW_INVALID_MESSAGE = `pagination window is too deep; page * size must be <= ${MAX_OFFSET_ROWS}`;
const SIZE_QUERY_INVALID_MESSAGE = 'size must be between 1 and 100';
const MAX_PAGE_SIZE = 100;
@@ -47,10 +49,14 @@ export class CollectionController {
return [...new Set(parsed)];
}
private validatePageQuery(page: number): void {
private validatePageQuery(page: number, size: number): void {
if (page < 0) {
throw new BadRequestException(PAGE_QUERY_INVALID_MESSAGE);
}
if (!isOffsetWithinLimit(page * size)) {
throw new BadRequestException(PAGE_WINDOW_INVALID_MESSAGE);
}
}
private validateSizeQuery(size: number): void {
@@ -146,8 +152,8 @@ export class CollectionController {
@Query('page', new DefaultValuePipe(0), ParseIntPipe) page: number,
@Query('size', new DefaultValuePipe(50), ParseIntPipe) size: number,
) {
this.validatePageQuery(page);
this.validateSizeQuery(size);
this.validatePageQuery(page, size);
return this.collectionService.getBooks(id, user, page, size);
}
}
server/src/modules/dashboard/dashboard.repository.test.ts
+19 -32
View File
@@ -1,6 +1,6 @@
import { DashboardRepository } from './dashboard.repository';
function makeCountChain<T>(rows: T) {
function makeBoundsChain<T>(rows: T) {
const chain: Record<string, vi.Mock> = {
from: vi.fn(),
leftJoin: vi.fn(),
@@ -31,24 +31,6 @@ function makeLimitChain<T>(rows: T) {
return chain;
}
function makeOffsetChain<T>(rows: T) {
const chain: Record<string, vi.Mock> = {
from: vi.fn(),
leftJoin: vi.fn(),
where: vi.fn(),
orderBy: vi.fn(),
limit: vi.fn(),
offset: vi.fn(),
};
chain.from.mockReturnValue(chain);
chain.leftJoin.mockReturnValue(chain);
chain.where.mockReturnValue(chain);
chain.orderBy.mockReturnValue(chain);
chain.limit.mockReturnValue(chain);
chain.offset.mockResolvedValue(rows);
return chain;
}
describe('DashboardRepository', () => {
beforeEach(() => {
vi.restoreAllMocks();
@@ -88,8 +70,8 @@ describe('DashboardRepository', () => {
});
it('returns empty random ids when there are no candidates', async () => {
const countChain = makeCountChain([{ total: '0' }]);
const db = { select: vi.fn().mockReturnValue(countChain) };
const boundsChain = makeBoundsChain([{ minId: null, maxId: null }]);
const db = { select: vi.fn().mockReturnValue(boundsChain) };
const repo = new DashboardRepository(db as never);
const result = await repo.findRandomBookIds([5], 7, 20);
@@ -99,33 +81,38 @@ describe('DashboardRepository', () => {
});
it('uses zero offset for random ids when total candidates are below requested limit', async () => {
const countChain = makeCountChain([{ total: '3' }]);
const listChain = makeOffsetChain([{ id: 1 }, { id: 2 }, { id: 3 }]);
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0);
const boundsChain = makeBoundsChain([{ minId: 1, maxId: 3 }]);
const firstPassChain = makeLimitChain([{ id: 1 }, { id: 2 }, { id: 3 }]);
const secondPassChain = makeLimitChain([]);
const db = {
select: vi.fn().mockReturnValueOnce(countChain).mockReturnValueOnce(listChain),
select: vi.fn().mockReturnValueOnce(boundsChain).mockReturnValueOnce(firstPassChain).mockReturnValueOnce(secondPassChain),
};
const repo = new DashboardRepository(db as never);
const result = await repo.findRandomBookIds([5], 7, 20);
expect(result).toEqual([1, 2, 3]);
expect(listChain.limit).toHaveBeenCalledWith(20);
expect(listChain.offset).toHaveBeenCalledWith(0);
expect(firstPassChain.limit).toHaveBeenCalledWith(20);
expect(secondPassChain.limit).toHaveBeenCalledWith(17);
randomSpy.mockRestore();
});
it('computes random offset within candidate range before reading random ids', async () => {
it('uses wrap-around sampling when anchored random pass does not fill the limit', async () => {
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5);
const countChain = makeCountChain([{ total: '12' }]);
const listChain = makeOffsetChain([{ id: 21 }, { id: 22 }]);
const boundsChain = makeBoundsChain([{ minId: 1, maxId: 12 }]);
const firstPassChain = makeLimitChain([{ id: 21 }, { id: 22 }]);
const secondPassChain = makeLimitChain([{ id: 1 }, { id: 2 }]);
const db = {
select: vi.fn().mockReturnValueOnce(countChain).mockReturnValueOnce(listChain),
select: vi.fn().mockReturnValueOnce(boundsChain).mockReturnValueOnce(firstPassChain).mockReturnValueOnce(secondPassChain),
};
const repo = new DashboardRepository(db as never);
const result = await repo.findRandomBookIds([3, 4], 99, 5);
expect(result).toEqual([21, 22]);
expect(listChain.offset).toHaveBeenCalledWith(4);
expect(result).toEqual([21, 22, 1, 2]);
expect(firstPassChain.limit).toHaveBeenCalledWith(5);
expect(secondPassChain.limit).toHaveBeenCalledWith(3);
randomSpy.mockRestore();
});
});
server/src/modules/dashboard/dashboard.repository.ts
+31 -13
View File
@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import { and, asc, count, desc, eq, gt, inArray, isNull, lt, or } from 'drizzle-orm';
import { and, desc, eq, gt, gte, inArray, isNull, lt, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db';
@@ -48,33 +48,51 @@ export class DashboardRepository {
async findRandomBookIds(accessibleLibraryIds: number[], userId: number, limit: number): Promise<number[]> {
if (accessibleLibraryIds.length === 0) return [];
if (limit <= 0) return [];
const unreadFilter = and(
inArray(books.libraryId, accessibleLibraryIds),
eq(books.status, 'present'),
or(isNull(readingProgress.bookFileId), eq(readingProgress.percentage, 0)),
);
const [{ total }] = await this.db
.select({ total: count() })
const [bounds] = await this.db
.select({
minId: sql<number | null>`min(${books.id})`,
maxId: sql<number | null>`max(${books.id})`,
})
.from(books)
.leftJoin(bookFiles, eq(bookFiles.id, books.primaryFileId))
.leftJoin(readingProgress, and(eq(readingProgress.bookFileId, bookFiles.id), eq(readingProgress.userId, userId)))
.where(unreadFilter);
const totalCandidates = Number(total);
if (totalCandidates === 0) return [];
if (bounds?.minId == null || bounds.maxId == null || bounds.minId > bounds.maxId) return [];
const maxOffset = Math.max(totalCandidates - limit, 0);
const offset = maxOffset === 0 ? 0 : Math.floor(Math.random() * (maxOffset + 1));
const range = bounds.maxId - bounds.minId + 1;
const anchorId = bounds.minId + Math.floor(Math.random() * range);
const rows = await this.db
const firstPass = await this.db
.select({ id: books.id })
.from(books)
.leftJoin(bookFiles, eq(bookFiles.id, books.primaryFileId))
.leftJoin(readingProgress, and(eq(readingProgress.bookFileId, bookFiles.id), eq(readingProgress.userId, userId)))
.where(unreadFilter)
.orderBy(asc(books.id))
.limit(limit)
.offset(offset);
return rows.map((row) => row.id);
.where(and(unreadFilter, gte(books.id, anchorId)))
.orderBy(books.id)
.limit(limit);
const remaining = limit - firstPass.length;
const secondPass =
remaining > 0
? await this.db
.select({ id: books.id })
.from(books)
.leftJoin(bookFiles, eq(bookFiles.id, books.primaryFileId))
.leftJoin(readingProgress, and(eq(readingProgress.bookFileId, bookFiles.id), eq(readingProgress.userId, userId)))
.where(and(unreadFilter, lt(books.id, anchorId)))
.orderBy(books.id)
.limit(remaining)
: [];
return [...firstPass, ...secondPass].map((row) => row.id);
}
}
server/src/modules/kobo/services/kobo-reading-state.service.ts
+48 -29
View File
@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import { and, eq, inArray, isNull, or, sql } from 'drizzle-orm';
import { and, asc, eq, inArray, isNull, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../../db';
@@ -8,6 +8,7 @@ import { KoboBookAccessService } from './kobo-book-access.service';
type Db = NodePgDatabase<typeof schema>;
type JsonObj = Record<string, unknown>;
const PROGRESS_PUSH_PAGE_SIZE = 250;
function mergeSubObject(incoming: JsonObj | null | undefined, existing: JsonObj | null | undefined): JsonObj | null {
if (!incoming) return existing ?? null;
@@ -121,7 +122,11 @@ export class KoboReadingStateService {
};
}
async getAndMarkStatesNeedingPush(userId: number, readingThreshold: number, finishedThreshold: number): Promise<unknown[]> {
async getAndMarkStatesNeedingPush(
userId: number,
readingThreshold: number,
finishedThreshold: number,
): Promise<{ items: unknown[]; hasMore: boolean }> {
const accessibleLibraryIds = await this.bookAccessService.getAccessibleLibraryIds(userId);
const libraryAccessFilter =
accessibleLibraryIds === null
@@ -148,14 +153,21 @@ export class KoboReadingStateService {
sql`${schema.readingProgress.updatedAt} > ${schema.koboReadingStates.progressSyncedAt}`,
),
),
);
)
.orderBy(asc(schema.readingProgress.updatedAt), asc(schema.books.id))
.limit(PROGRESS_PUSH_PAGE_SIZE + 1);
if (rows.length === 0) return [];
if (rows.length === 0) return { items: [], hasMore: false };
const now = new Date().toISOString();
const hasMore = rows.length > PROGRESS_PUSH_PAGE_SIZE;
const pageRows = rows.slice(0, PROGRESS_PUSH_PAGE_SIZE);
const now = new Date();
const nowIso = now.toISOString();
const result: unknown[] = [];
const upsertRows: (typeof schema.koboReadingStates.$inferInsert)[] = [];
for (const row of rows) {
for (const row of pageRows) {
const id = String(row.bookId);
const pct = row.percentage ?? 0;
const status = pct >= finishedThreshold ? 'Finished' : pct >= readingThreshold ? 'Reading' : 'ReadyToRead';
@@ -164,35 +176,42 @@ export class KoboReadingStateService {
ChangedReadingState: {
ReadingState: {
EntitlementId: id,
Created: now,
LastModified: now,
PriorityTimestamp: now,
StatusInfo: { LastModified: now, Status: status, TimesStartedReading: pct >= readingThreshold ? 1 : 0 },
Statistics: { LastModified: now },
CurrentBookmark: { LastModified: now, ProgressPercent: pct, ContentSourceProgressPercent: pct },
Created: nowIso,
LastModified: nowIso,
PriorityTimestamp: nowIso,
StatusInfo: { LastModified: nowIso, Status: status, TimesStartedReading: pct >= readingThreshold ? 1 : 0 },
Statistics: { LastModified: nowIso },
CurrentBookmark: { LastModified: nowIso, ProgressPercent: pct, ContentSourceProgressPercent: pct },
},
},
});
await this.db
.insert(schema.koboReadingStates)
.values({
userId,
bookId: row.bookId,
entitlementId: id,
createdAtKobo: now,
lastModifiedKobo: now,
priorityTimestamp: now,
progressSyncedAt: new Date(),
updatedAt: new Date(),
})
.onConflictDoUpdate({
target: [schema.koboReadingStates.userId, schema.koboReadingStates.bookId],
set: { progressSyncedAt: sql`now()` },
});
upsertRows.push({
userId,
bookId: row.bookId,
entitlementId: id,
createdAtKobo: nowIso,
lastModifiedKobo: nowIso,
priorityTimestamp: nowIso,
progressSyncedAt: now,
updatedAt: now,
});
}
return result;
await this.db
.insert(schema.koboReadingStates)
.values(upsertRows)
.onConflictDoUpdate({
target: [schema.koboReadingStates.userId, schema.koboReadingStates.bookId],
set: {
lastModifiedKobo: nowIso,
priorityTimestamp: nowIso,
progressSyncedAt: sql`now()`,
updatedAt: sql`now()`,
},
});
return { items: result, hasMore };
}
private extractPercent(bookmark: JsonObj | null): number | null {
server/src/modules/kobo/services/kobo-sync.service.ts
+264 -157
View File
@@ -1,6 +1,6 @@
import { createHash } from 'crypto';
import { Inject, Injectable } from '@nestjs/common';
import { and, eq, inArray, sql } from 'drizzle-orm';
import { SQL, and, asc, eq, inArray, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../../db/db.module';
@@ -11,8 +11,15 @@ import { KoboReadingStateService } from './kobo-reading-state.service';
type Db = NodePgDatabase<typeof schema>;
const SYNC_PAGE_SIZE = 5;
const SNAPSHOT_RECONCILE_BATCH_SIZE = 5000;
const TOKEN_PREFIX = 'PX.';
type EligibleSnapshotRow = {
bookId: number;
fileHash: string | null;
metadataHash: string;
};
export interface KoboBookEntry {
bookId: number;
title: string;
@@ -62,20 +69,20 @@ export class KoboSyncService {
where: eq(schema.koboLibrarySnapshots.userId, userId),
});
const eligibleBooks = await this.fetchEligibleBooks(userId);
const eligibleIds = eligibleBooks.map((b) => b.bookId);
const eligibleSnapshotRows = await this.fetchEligibleSnapshotRows(userId);
if (!snapshot) {
await this.createSnapshot(userId, eligibleBooks);
await this.createSnapshot(userId, eligibleSnapshotRows);
} else {
await this.reconcileSnapshot(snapshot.id, eligibleIds, eligibleBooks);
await this.reconcileSnapshot(snapshot.id, eligibleSnapshotRows);
}
return this.getPageFromSnapshot(userId, deviceToken, baseUrl, settings);
return this.getPageFromSnapshot(userId, deviceToken, baseUrl, settings, new Set(eligibleSnapshotRows.map((row) => row.bookId)));
}
async getBookMetadata(userId: number, bookId: number, deviceToken: string, baseUrl: string): Promise<unknown[]> {
const book = await this.fetchBookById(bookId, userId);
const booksById = await this.fetchEligibleBooksByIds(userId, [bookId]);
const book = booksById.get(bookId) ?? null;
if (!book) return [];
return [this.buildBookMetadata(book, deviceToken, baseUrl)];
}
@@ -111,69 +118,111 @@ export class KoboSyncService {
await this.db.update(schema.koboSnapshotBooks).set({ synced: false }).where(eq(schema.koboSnapshotBooks.snapshotId, snapshot.id));
}
private async createSnapshot(userId: number, books: KoboBookEntry[]) {
private async createSnapshot(userId: number, books: EligibleSnapshotRow[]) {
const [snap] = await this.db.insert(schema.koboLibrarySnapshots).values({ userId }).returning();
if (books.length > 0) {
await this.db.insert(schema.koboSnapshotBooks).values(
books.map((b) => ({
snapshotId: snap.id,
bookId: b.bookId,
synced: false,
pendingDelete: false,
isNew: true,
removedByDevice: false,
fileHash: b.fileHash,
metadataHash: b.metadataHash,
})),
);
await this.db
.insert(schema.koboSnapshotBooks)
.values(
books.map((b) => ({
snapshotId: snap.id,
bookId: b.bookId,
synced: false,
pendingDelete: false,
isNew: true,
removedByDevice: false,
fileHash: b.fileHash,
metadataHash: b.metadataHash,
})),
)
.onConflictDoNothing();
}
}
private async reconcileSnapshot(snapshotId: number, eligibleIds: number[], eligibleBooks: KoboBookEntry[]) {
const existing = await this.db.select().from(schema.koboSnapshotBooks).where(eq(schema.koboSnapshotBooks.snapshotId, snapshotId));
private async reconcileSnapshot(snapshotId: number, eligibleBooks: EligibleSnapshotRow[]) {
await this.db.transaction(async (tx) => {
await tx.execute(sql`
CREATE TEMP TABLE kobo_eligible_books_tmp (
book_id integer PRIMARY KEY,
file_hash varchar(64),
metadata_hash varchar(64) NOT NULL
) ON COMMIT DROP
`);
const existingMap = new Map(existing.map((r) => [r.bookId, r]));
const eligibleMap = new Map(eligibleBooks.map((b) => [b.bookId, b]));
for (let index = 0; index < eligibleBooks.length; index += SNAPSHOT_RECONCILE_BATCH_SIZE) {
const chunk = eligibleBooks.slice(index, index + SNAPSHOT_RECONCILE_BATCH_SIZE);
const bookIds = chunk.map((book) => book.bookId);
const fileHashes = chunk.map((book) => book.fileHash);
const metadataHashes = chunk.map((book) => book.metadataHash);
for (const row of existing) {
if (!eligibleMap.has(row.bookId) && !row.pendingDelete) {
if (row.removedByDevice) {
await this.db
.delete(schema.koboSnapshotBooks)
.where(and(eq(schema.koboSnapshotBooks.snapshotId, snapshotId), eq(schema.koboSnapshotBooks.bookId, row.bookId)));
} else if (row.synced) {
await this.db
.update(schema.koboSnapshotBooks)
.set({ pendingDelete: true, synced: false })
.where(and(eq(schema.koboSnapshotBooks.snapshotId, snapshotId), eq(schema.koboSnapshotBooks.bookId, row.bookId)));
} else {
await this.db
.delete(schema.koboSnapshotBooks)
.where(and(eq(schema.koboSnapshotBooks.snapshotId, snapshotId), eq(schema.koboSnapshotBooks.bookId, row.bookId)));
}
await tx.execute(sql`
INSERT INTO kobo_eligible_books_tmp (book_id, file_hash, metadata_hash)
SELECT * FROM unnest(${bookIds}::integer[], ${fileHashes}::text[], ${metadataHashes}::text[])
ON CONFLICT (book_id)
DO UPDATE
SET file_hash = EXCLUDED.file_hash,
metadata_hash = EXCLUDED.metadata_hash
`);
}
}
for (const book of eligibleBooks) {
const snap = existingMap.get(book.bookId);
if (!snap) {
await this.db.insert(schema.koboSnapshotBooks).values({
snapshotId,
bookId: book.bookId,
synced: false,
pendingDelete: false,
isNew: true,
removedByDevice: false,
fileHash: book.fileHash,
metadataHash: book.metadataHash,
});
} else if (!snap.removedByDevice && snap.synced && (snap.fileHash !== book.fileHash || snap.metadataHash !== book.metadataHash)) {
await this.db
.update(schema.koboSnapshotBooks)
.set({ synced: false, isNew: false, fileHash: book.fileHash, metadataHash: book.metadataHash })
.where(and(eq(schema.koboSnapshotBooks.snapshotId, snapshotId), eq(schema.koboSnapshotBooks.bookId, book.bookId)));
}
}
await tx.execute(sql`
UPDATE ${schema.koboSnapshotBooks} AS sb
SET pending_delete = true,
synced = false
WHERE sb.snapshot_id = ${snapshotId}
AND sb.pending_delete = false
AND sb.removed_by_device = false
AND sb.synced = true
AND NOT EXISTS (SELECT 1 FROM kobo_eligible_books_tmp e WHERE e.book_id = sb.book_id)
`);
await tx.execute(sql`
DELETE FROM ${schema.koboSnapshotBooks} AS sb
WHERE sb.snapshot_id = ${snapshotId}
AND sb.pending_delete = false
AND (sb.removed_by_device = true OR sb.synced = false)
AND NOT EXISTS (SELECT 1 FROM kobo_eligible_books_tmp e WHERE e.book_id = sb.book_id)
`);
await tx.execute(sql`
INSERT INTO ${schema.koboSnapshotBooks}
(snapshot_id, book_id, synced, pending_delete, is_new, removed_by_device, file_hash, metadata_hash)
SELECT ${snapshotId}, e.book_id, false, false, true, false, e.file_hash, e.metadata_hash
FROM kobo_eligible_books_tmp e
LEFT JOIN ${schema.koboSnapshotBooks} sb
ON sb.snapshot_id = ${snapshotId}
AND sb.book_id = e.book_id
WHERE sb.book_id IS NULL
`);
await tx.execute(sql`
UPDATE ${schema.koboSnapshotBooks} AS sb
SET pending_delete = false,
synced = false,
is_new = true,
file_hash = e.file_hash,
metadata_hash = e.metadata_hash
FROM kobo_eligible_books_tmp e
WHERE sb.snapshot_id = ${snapshotId}
AND sb.book_id = e.book_id
AND sb.pending_delete = true
AND sb.removed_by_device = false
`);
await tx.execute(sql`
UPDATE ${schema.koboSnapshotBooks} AS sb
SET synced = false,
is_new = false,
file_hash = e.file_hash,
metadata_hash = e.metadata_hash
FROM kobo_eligible_books_tmp e
WHERE sb.snapshot_id = ${snapshotId}
AND sb.book_id = e.book_id
AND sb.removed_by_device = false
AND sb.synced = true
AND (sb.file_hash IS DISTINCT FROM e.file_hash OR sb.metadata_hash IS DISTINCT FROM e.metadata_hash)
`);
});
}
private async getPageFromSnapshot(
@@ -181,6 +230,7 @@ export class KoboSyncService {
deviceToken: string,
baseUrl: string,
settings: SyncSettings,
eligibleIds: Set<number>,
): Promise<{ entitlements: unknown[]; hasMore: boolean; syncToken: string }> {
const snapshot = await this.db.query.koboLibrarySnapshots.findFirst({
where: eq(schema.koboLibrarySnapshots.userId, userId),
@@ -194,6 +244,7 @@ export class KoboSyncService {
.select()
.from(schema.koboSnapshotBooks)
.where(and(eq(schema.koboSnapshotBooks.snapshotId, snapshot.id), eq(schema.koboSnapshotBooks.synced, false)))
.orderBy(asc(schema.koboSnapshotBooks.bookId))
.limit(SYNC_PAGE_SIZE + 1);
const hasMore = pending.length > SYNC_PAGE_SIZE;
@@ -204,10 +255,13 @@ export class KoboSyncService {
if (settings.twoWayProgressSync) {
const pushed = await this.readingStateService.getAndMarkStatesNeedingPush(userId, settings.readingThreshold, settings.finishedThreshold);
endItems.push(...pushed);
endItems.push(...pushed.items);
if (pushed.hasMore) {
return { entitlements: endItems, hasMore: true, syncToken };
}
}
const tagItems = await this.buildTagItems(userId);
const tagItems = await this.buildTagItems(userId, eligibleIds);
endItems.push(...tagItems);
return { entitlements: endItems, hasMore: false, syncToken };
@@ -220,6 +274,10 @@ export class KoboSyncService {
.where(and(eq(schema.koboSnapshotBooks.snapshotId, snapshot.id), inArray(schema.koboSnapshotBooks.bookId, pageIds)));
const entitlements: unknown[] = [];
const booksById = await this.fetchEligibleBooksByIds(
userId,
page.filter((row) => !row.pendingDelete).map((row) => row.bookId),
);
for (const row of page) {
if (row.pendingDelete) {
@@ -230,7 +288,7 @@ export class KoboSyncService {
continue;
}
const book = await this.fetchBookById(row.bookId, userId);
const book = booksById.get(row.bookId) ?? null;
if (!book) continue;
if (row.isNew) {
@@ -255,15 +313,13 @@ export class KoboSyncService {
return { entitlements, hasMore, syncToken };
}
private async buildTagItems(userId: number): Promise<unknown[]> {
private async buildTagItems(userId: number, eligibleIds: Set<number>): Promise<unknown[]> {
const collections = await this.db.query.collections.findMany({
where: and(eq(schema.collections.userId, userId), eq(schema.collections.syncToKobo, true)),
});
if (collections.length === 0) return [];
const collectionIds = collections.map((c) => c.id);
const eligibleBooks = await this.fetchEligibleBooks(userId);
const eligibleIds = new Set(eligibleBooks.map((b) => b.bookId));
const collectionBooks = await this.db
.select({ collectionId: schema.collectionBooks.collectionId, bookId: schema.collectionBooks.bookId })
@@ -402,7 +458,24 @@ export class KoboSyncService {
};
}
private async fetchEligibleBooks(userId: number): Promise<KoboBookEntry[]> {
private buildMetadataHash(params: {
title: string | null;
authors: string[];
seriesName: string | null;
seriesIndex: number | null;
metadataUpdatedAt: Date | null;
}): string {
const metaStr = [
params.title ?? '',
params.authors.join(','),
params.seriesName ?? '',
String(params.seriesIndex ?? ''),
String(params.metadataUpdatedAt ?? ''),
].join('|');
return createHash('sha256').update(metaStr).digest('hex').slice(0, 16);
}
private async buildEligibleBooksWhereClause(userId: number): Promise<SQL | undefined> {
const accessibleLibraryIds = await this.bookAccessService.getAccessibleLibraryIds(userId);
const libraryAccessFilter =
accessibleLibraryIds === null
@@ -411,6 +484,68 @@ export class KoboSyncService {
? sql`false`
: inArray(schema.books.libraryId, accessibleLibraryIds);
const collectionMembershipFilter = sql`EXISTS (
SELECT 1
FROM ${schema.collectionBooks}
INNER JOIN ${schema.collections}
ON ${schema.collections.id} = ${schema.collectionBooks.collectionId}
WHERE ${schema.collectionBooks.bookId} = ${schema.books.id}
AND ${schema.collections.userId} = ${userId}
AND ${schema.collections.syncToKobo} = true
)`;
return and(eq(schema.books.status, 'present'), eq(schema.bookFiles.format, 'epub'), libraryAccessFilter, collectionMembershipFilter);
}
private async fetchEligibleSnapshotRows(userId: number): Promise<EligibleSnapshotRow[]> {
const whereClause = await this.buildEligibleBooksWhereClause(userId);
if (!whereClause) return [];
const rows = await this.db
.select({
bookId: schema.books.id,
title: schema.bookMetadata.title,
seriesName: schema.bookMetadata.seriesName,
seriesIndex: schema.bookMetadata.seriesIndex,
metadataUpdatedAt: schema.bookMetadata.updatedAt,
fileHash: schema.bookFiles.hash,
authorNamesCsv: sql<string>`coalesce(string_agg(${schema.authors.name}, ',' ORDER BY ${schema.bookAuthors.displayOrder}, ${schema.bookAuthors.authorId}), '')`,
})
.from(schema.books)
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.books.primaryFileId))
.leftJoin(schema.bookMetadata, eq(schema.bookMetadata.bookId, schema.books.id))
.leftJoin(schema.bookAuthors, eq(schema.bookAuthors.bookId, schema.books.id))
.leftJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(whereClause)
.groupBy(
schema.books.id,
schema.bookMetadata.title,
schema.bookMetadata.seriesName,
schema.bookMetadata.seriesIndex,
schema.bookMetadata.updatedAt,
schema.bookFiles.hash,
);
return rows.map((row) => ({
bookId: row.bookId,
fileHash: row.fileHash,
metadataHash: this.buildMetadataHash({
title: row.title,
authors: row.authorNamesCsv ? row.authorNamesCsv.split(',').filter((name) => name.length > 0) : [],
seriesName: row.seriesName,
seriesIndex: row.seriesIndex,
metadataUpdatedAt: row.metadataUpdatedAt,
}),
}));
}
private async fetchEligibleBooksByIds(userId: number, bookIds: number[]): Promise<Map<number, KoboBookEntry>> {
const uniqueBookIds = [...new Set(bookIds)];
if (uniqueBookIds.length === 0) return new Map();
const whereClause = await this.buildEligibleBooksWhereClause(userId);
if (!whereClause) return new Map();
const rows = await this.db
.select({
bookId: schema.books.id,
@@ -431,104 +566,76 @@ export class KoboSyncService {
.from(schema.books)
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.books.primaryFileId))
.leftJoin(schema.bookMetadata, eq(schema.bookMetadata.bookId, schema.books.id))
.innerJoin(schema.collectionBooks, eq(schema.collectionBooks.bookId, schema.books.id))
.innerJoin(
schema.collections,
and(
eq(schema.collections.id, schema.collectionBooks.collectionId),
eq(schema.collections.userId, userId),
eq(schema.collections.syncToKobo, true),
),
)
.where(and(eq(schema.books.status, 'present'), sql`lower(${schema.bookFiles.format}) in ('epub')`, libraryAccessFilter))
.groupBy(
schema.books.id,
schema.bookMetadata.title,
schema.bookMetadata.description,
schema.bookMetadata.publisher,
schema.bookMetadata.publishedYear,
schema.bookMetadata.language,
schema.bookMetadata.seriesName,
schema.bookMetadata.seriesIndex,
schema.bookFiles.format,
schema.bookFiles.sizeBytes,
schema.bookFiles.hash,
schema.bookMetadata.updatedAt,
schema.books.addedAt,
schema.books.updatedAt,
);
.where(and(whereClause, inArray(schema.books.id, uniqueBookIds)));
if (rows.length === 0) return [];
if (rows.length === 0) return new Map();
const bookIds = rows.map((r) => r.bookId);
const authorRows = await this.db
.select({ bookId: schema.bookAuthors.bookId, name: schema.authors.name })
.from(schema.bookAuthors)
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(inArray(schema.bookAuthors.bookId, bookIds))
.orderBy(schema.bookAuthors.displayOrder);
const collectionRows = await this.db
.select({ bookId: schema.collectionBooks.bookId, name: schema.collections.name })
.from(schema.collectionBooks)
.innerJoin(
schema.collections,
and(
eq(schema.collections.id, schema.collectionBooks.collectionId),
eq(schema.collections.userId, userId),
eq(schema.collections.syncToKobo, true),
),
)
.where(inArray(schema.collectionBooks.bookId, bookIds));
const fetchedBookIds = rows.map((row) => row.bookId);
const [authorRows, collectionRows] = await Promise.all([
this.db
.select({ bookId: schema.bookAuthors.bookId, name: schema.authors.name })
.from(schema.bookAuthors)
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(inArray(schema.bookAuthors.bookId, fetchedBookIds))
.orderBy(schema.bookAuthors.displayOrder),
this.db
.select({ bookId: schema.collectionBooks.bookId, name: schema.collections.name })
.from(schema.collectionBooks)
.innerJoin(
schema.collections,
and(
eq(schema.collections.id, schema.collectionBooks.collectionId),
eq(schema.collections.userId, userId),
eq(schema.collections.syncToKobo, true),
),
)
.where(inArray(schema.collectionBooks.bookId, fetchedBookIds)),
]);
const authorsByBook = new Map<number, string[]>();
for (const r of authorRows) {
const list = authorsByBook.get(r.bookId) ?? [];
list.push(r.name);
authorsByBook.set(r.bookId, list);
for (const row of authorRows) {
const names = authorsByBook.get(row.bookId) ?? [];
names.push(row.name);
authorsByBook.set(row.bookId, names);
}
const collectionsByBook = new Map<number, string[]>();
for (const r of collectionRows) {
const list = collectionsByBook.get(r.bookId) ?? [];
if (!list.includes(r.name)) list.push(r.name);
collectionsByBook.set(r.bookId, list);
for (const row of collectionRows) {
const names = collectionsByBook.get(row.bookId) ?? [];
if (!names.includes(row.name)) names.push(row.name);
collectionsByBook.set(row.bookId, names);
}
return rows.map((r) => {
const metaStr = [
r.title ?? '',
(authorsByBook.get(r.bookId) ?? []).join(','),
r.seriesName ?? '',
String(r.seriesIndex ?? ''),
String(r.metadataUpdatedAt ?? ''),
].join('|');
const metadataHash = createHash('sha256').update(metaStr).digest('hex').slice(0, 16);
return {
bookId: r.bookId,
title: r.title ?? `Book ${r.bookId}`,
authors: authorsByBook.get(r.bookId) ?? [],
description: r.description,
publisher: r.publisher,
publishedYear: r.publishedYear,
language: r.language,
seriesName: r.seriesName,
seriesIndex: r.seriesIndex,
fileFormat: r.fileFormat ?? 'epub',
fileSizeBytes: r.fileSizeBytes,
fileHash: r.fileHash,
metadataHash,
metadataUpdatedAt: r.metadataUpdatedAt,
collectionNames: collectionsByBook.get(r.bookId) ?? [],
addedAt: r.addedAt,
updatedAt: r.updatedAt,
};
});
}
const byId = new Map<number, KoboBookEntry>();
for (const row of rows) {
const authors = authorsByBook.get(row.bookId) ?? [];
byId.set(row.bookId, {
bookId: row.bookId,
title: row.title ?? `Book ${row.bookId}`,
authors,
description: row.description,
publisher: row.publisher,
publishedYear: row.publishedYear,
language: row.language,
seriesName: row.seriesName,
seriesIndex: row.seriesIndex,
fileFormat: row.fileFormat ?? 'epub',
fileSizeBytes: row.fileSizeBytes,
fileHash: row.fileHash,
metadataHash: this.buildMetadataHash({
title: row.title,
authors,
seriesName: row.seriesName,
seriesIndex: row.seriesIndex,
metadataUpdatedAt: row.metadataUpdatedAt,
}),
metadataUpdatedAt: row.metadataUpdatedAt,
collectionNames: collectionsByBook.get(row.bookId) ?? [],
addedAt: row.addedAt,
updatedAt: row.updatedAt,
});
}
private async fetchBookById(bookId: number, userId: number): Promise<KoboBookEntry | null> {
const books = await this.fetchEligibleBooks(userId);
return books.find((b) => b.bookId === bookId) ?? null;
return byId;
}
}
server/src/modules/metadata/metadata.service.test.ts
+20 -43
View File
@@ -376,21 +376,14 @@ describe('MetadataService', () => {
const insertedAuthors: Array<{ name: string; sortName: string | null }> = [];
const insertedBookAuthors: Array<{ bookId: number; authorId: number; displayOrder: number }> = [];
db.select.mockImplementation(() => ({
from: () => ({
where: () => ({
limit: () => Promise.resolve([]),
}),
}),
}));
db.insert.mockImplementation((table: unknown) => {
if (table === authors) {
return {
values: (row: { name: string; sortName: string | null }) => {
insertedAuthors.push(row);
values: (rows: Array<{ name: string; sortName: string | null }>) => {
insertedAuthors.push(...rows);
return {
onConflictDoNothing: () => ({
returning: () => Promise.resolve([{ id: 81 }]),
returning: () => Promise.resolve([{ id: 81, name: 'Alice' }]),
}),
};
},
@@ -398,8 +391,8 @@ describe('MetadataService', () => {
}
if (table === bookAuthors) {
return {
values: (row: { bookId: number; authorId: number; displayOrder: number }) => {
insertedBookAuthors.push(row);
values: (rows: Array<{ bookId: number; authorId: number; displayOrder: number }>) => {
insertedBookAuthors.push(...rows);
return { onConflictDoNothing: () => Promise.resolve(undefined) };
},
};
@@ -429,16 +422,14 @@ describe('MetadataService', () => {
db.select.mockImplementation(() => ({
from: () => ({
where: () => ({
limit: () => Promise.resolve([{ id: 9 }]),
}),
where: () => Promise.resolve([{ id: 9, name: 'Known Author' }]),
}),
}));
db.insert.mockImplementation((table: unknown) => {
if (table === authors) {
return {
values: (row: { name: string; sortName: string | null }) => {
insertedAuthors.push(row);
values: (rows: Array<{ name: string; sortName: string | null }>) => {
insertedAuthors.push(...rows);
return {
onConflictDoNothing: () => ({
returning: () => Promise.resolve([]),
@@ -449,8 +440,8 @@ describe('MetadataService', () => {
}
if (table === bookAuthors) {
return {
values: (row: { bookId: number; authorId: number; displayOrder: number }) => {
insertedBookAuthors.push(row);
values: (rows: Array<{ bookId: number; authorId: number; displayOrder: number }>) => {
insertedBookAuthors.push(...rows);
return { onConflictDoNothing: () => Promise.resolve(undefined) };
},
};
@@ -481,7 +472,7 @@ describe('MetadataService', () => {
return {
values: () => ({
onConflictDoNothing: () => ({
returning: () => Promise.resolve([{ id: 81 }]),
returning: () => Promise.resolve([{ id: 81, name: 'New Author' }]),
}),
}),
};
@@ -507,21 +498,14 @@ describe('MetadataService', () => {
const insertedGenres: string[] = [];
const insertedBookGenres: Array<{ bookId: number; genreId: number }> = [];
db.select.mockImplementation(() => ({
from: () => ({
where: () => ({
limit: () => Promise.resolve([]),
}),
}),
}));
db.insert.mockImplementation((table: unknown) => {
if (table === genres) {
return {
values: (row: { name: string }) => {
insertedGenres.push(row.name);
values: (rows: Array<{ name: string }>) => {
insertedGenres.push(...rows.map((row) => row.name));
return {
onConflictDoNothing: () => ({
returning: () => Promise.resolve([{ id: 77 }]),
returning: () => Promise.resolve(rows.map((row, index) => ({ id: 77 + index, name: row.name }))),
}),
};
},
@@ -529,8 +513,8 @@ describe('MetadataService', () => {
}
if (table === bookGenres) {
return {
values: (row: { bookId: number; genreId: number }) => {
insertedBookGenres.push(row);
values: (rows: Array<{ bookId: number; genreId: number }>) => {
insertedBookGenres.push(...rows);
return { onConflictDoNothing: () => Promise.resolve(undefined) };
},
};
@@ -545,7 +529,7 @@ describe('MetadataService', () => {
expect(insertedGenres).toEqual(['Fantasy', 'X'.repeat(200)]);
expect(insertedBookGenres).toEqual([
{ bookId: 12, genreId: 77 },
{ bookId: 12, genreId: 77 },
{ bookId: 12, genreId: 78 },
]);
});
@@ -554,21 +538,14 @@ describe('MetadataService', () => {
const service = makeService(db);
const insertedTags: string[] = [];
db.select.mockImplementation(() => ({
from: () => ({
where: () => ({
limit: () => Promise.resolve([]),
}),
}),
}));
db.insert.mockImplementation((table: unknown) => {
if (table === tags) {
return {
values: (row: { name: string }) => {
insertedTags.push(row.name);
values: (rows: Array<{ name: string }>) => {
insertedTags.push(...rows.map((row) => row.name));
return {
onConflictDoNothing: () => ({
returning: () => Promise.resolve([{ id: 88 }]),
returning: () => Promise.resolve(rows.map((row, index) => ({ id: 88 + index, name: row.name }))),
}),
};
},
server/src/modules/metadata/metadata.service.ts
+80 -27
View File
@@ -356,35 +356,70 @@ export class MetadataService {
if (uniqueAuthors.length === 0) return [];
const authorIds: number[] = [];
for (let index = 0; index < uniqueAuthors.length; index += 1) {
const { name, sortName } = uniqueAuthors[index];
let [author] = await executor.insert(authors).values({ name, sortName }).onConflictDoNothing().returning();
if (!author) {
[author] = await executor.select().from(authors).where(eq(authors.name, name)).limit(1);
}
if (!author) continue;
await executor.insert(bookAuthors).values({ bookId, authorId: author.id, displayOrder: index }).onConflictDoNothing();
authorIds.push(author.id);
const authorByName = new Map<string, { id: number }>();
const insertedAuthors = await executor
.insert(authors)
.values(uniqueAuthors.map((author) => ({ name: author.name, sortName: author.sortName })))
.onConflictDoNothing()
.returning({ id: authors.id, name: authors.name });
for (const row of insertedAuthors) {
authorByName.set(row.name, { id: row.id });
}
return authorIds;
const unresolvedNames = [...new Set(uniqueAuthors.map((author) => author.name))].filter((name) => !authorByName.has(name));
if (unresolvedNames.length > 0) {
const existingAuthors = await executor
.select({ id: authors.id, name: authors.name })
.from(authors)
.where(inArray(authors.name, unresolvedNames));
for (const row of existingAuthors) {
authorByName.set(row.name, { id: row.id });
}
}
const links = uniqueAuthors.flatMap((author, index) => {
const match = authorByName.get(author.name);
if (!match) return [];
return [{ bookId, authorId: match.id, displayOrder: index }];
});
if (links.length > 0) {
await executor.insert(bookAuthors).values(links).onConflictDoNothing();
}
return links.map((link) => link.authorId);
}
private async replaceGenresInExecutor(executor: RelationMutationExecutor, bookId: number, uniqueGenres: string[]): Promise<void> {
await executor.delete(bookGenres).where(eq(bookGenres.bookId, bookId));
if (uniqueGenres.length === 0) return;
for (const genreName of uniqueGenres) {
let [genre] = await executor.insert(genres).values({ name: genreName }).onConflictDoNothing().returning();
if (!genre) {
[genre] = await executor.select().from(genres).where(eq(genres.name, genreName)).limit(1);
}
if (!genre) continue;
const genreByName = new Map<string, { id: number }>();
const insertedGenres = await executor
.insert(genres)
.values(uniqueGenres.map((name) => ({ name })))
.onConflictDoNothing()
.returning({ id: genres.id, name: genres.name });
for (const row of insertedGenres) {
genreByName.set(row.name, { id: row.id });
}
await executor.insert(bookGenres).values({ bookId, genreId: genre.id }).onConflictDoNothing();
const unresolvedNames = uniqueGenres.filter((name) => !genreByName.has(name));
if (unresolvedNames.length > 0) {
const existingGenres = await executor.select({ id: genres.id, name: genres.name }).from(genres).where(inArray(genres.name, unresolvedNames));
for (const row of existingGenres) {
genreByName.set(row.name, { id: row.id });
}
}
const links = uniqueGenres.flatMap((name) => {
const match = genreByName.get(name);
if (!match) return [];
return [{ bookId, genreId: match.id }];
});
if (links.length > 0) {
await executor.insert(bookGenres).values(links).onConflictDoNothing();
}
}
@@ -392,14 +427,32 @@ export class MetadataService {
await executor.delete(bookTags).where(eq(bookTags.bookId, bookId));
if (uniqueTags.length === 0) return;
for (const tagName of uniqueTags) {
let [tag] = await executor.insert(tags).values({ name: tagName }).onConflictDoNothing().returning();
if (!tag) {
[tag] = await executor.select().from(tags).where(eq(tags.name, tagName)).limit(1);
}
if (!tag) continue;
const tagByName = new Map<string, { id: number }>();
const insertedTags = await executor
.insert(tags)
.values(uniqueTags.map((name) => ({ name })))
.onConflictDoNothing()
.returning({ id: tags.id, name: tags.name });
for (const row of insertedTags) {
tagByName.set(row.name, { id: row.id });
}
await executor.insert(bookTags).values({ bookId, tagId: tag.id }).onConflictDoNothing();
const unresolvedNames = uniqueTags.filter((name) => !tagByName.has(name));
if (unresolvedNames.length > 0) {
const existingTags = await executor.select({ id: tags.id, name: tags.name }).from(tags).where(inArray(tags.name, unresolvedNames));
for (const row of existingTags) {
tagByName.set(row.name, { id: row.id });
}
}
const links = uniqueTags.flatMap((name) => {
const match = tagByName.get(name);
if (!match) return [];
return [{ bookId, tagId: match.id }];
});
if (links.length > 0) {
await executor.insert(bookTags).values(links).onConflictDoNothing();
}
}
server/src/modules/migration/planner/matching.service.ts
+29 -26
View File
@@ -1,5 +1,5 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { and, eq, ilike, inArray, sql } from 'drizzle-orm';
import { and, eq, ilike, inArray, or, sql } from 'drizzle-orm';
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../../db';
@@ -246,35 +246,38 @@ export class MatchingService {
const normalizedTitle = title.trim();
if (!normalizedTitle) return NONE;
const normalizedAuthors = authors.map((author) => normalizeAuthor(author)).filter((author): author is string => !!author);
const normalizedAuthors = [...new Set(authors.map((author) => normalizeAuthor(author)).filter((author): author is string => !!author))];
if (normalizedAuthors.length === 0) return NONE;
for (const normalizedAuthor of normalizedAuthors) {
const exactRows = await this.db
.select({ bookId: schema.bookMetadata.bookId })
.from(schema.bookMetadata)
.innerJoin(schema.bookAuthors, eq(schema.bookAuthors.bookId, schema.bookMetadata.bookId))
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(
and(sql`lower(${schema.bookMetadata.title}) = lower(${normalizedTitle})`, sql`lower(${schema.authors.name}) = lower(${normalizedAuthor})`),
)
.limit(2);
const result = toLookupResult(exactRows);
if (result.kind !== 'none') return result;
const normalizedAuthorsLower = normalizedAuthors.map((author) => author.toLowerCase());
const exactRows = await this.db
.selectDistinct({ bookId: schema.bookMetadata.bookId })
.from(schema.bookMetadata)
.innerJoin(schema.bookAuthors, eq(schema.bookAuthors.bookId, schema.bookMetadata.bookId))
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(
and(
sql`lower(${schema.bookMetadata.title}) = lower(${normalizedTitle})`,
inArray(sql<string>`lower(${schema.authors.name})`, normalizedAuthorsLower),
),
)
.limit(2);
const exactResult = toLookupResult(exactRows);
if (exactResult.kind !== 'none') {
return exactResult;
}
for (const normalizedAuthor of normalizedAuthors) {
const approxRows = await this.db
.select({ bookId: schema.bookMetadata.bookId })
.from(schema.bookMetadata)
.innerJoin(schema.bookAuthors, eq(schema.bookAuthors.bookId, schema.bookMetadata.bookId))
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(
and(sql`lower(${schema.bookMetadata.title}) = lower(${normalizedTitle})`, ilike(schema.authors.name, `%${escapeLike(normalizedAuthor)}%`)),
)
.limit(2);
const result = toLookupResult(approxRows);
if (result.kind !== 'none') return result;
const approxClauses = normalizedAuthors.map((author) => ilike(schema.authors.name, `%${escapeLike(author)}%`));
const approxRows = await this.db
.selectDistinct({ bookId: schema.bookMetadata.bookId })
.from(schema.bookMetadata)
.innerJoin(schema.bookAuthors, eq(schema.bookAuthors.bookId, schema.bookMetadata.bookId))
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(and(sql`lower(${schema.bookMetadata.title}) = lower(${normalizedTitle})`, or(...approxClauses)!))
.limit(2);
const approxResult = toLookupResult(approxRows);
if (approxResult.kind !== 'none') {
return approxResult;
}
return NONE;
server/src/modules/narrator/narrator.repository.ts
+31 -12
View File
@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import { eq } from 'drizzle-orm';
import { eq, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db';
@@ -31,19 +31,38 @@ export class NarratorRepository {
): Promise<void> {
await executor.delete(bookNarrators).where(eq(bookNarrators.bookId, bookId));
for (let i = 0; i < names.length; i++) {
const { name, sortName } = names[i];
if (names.length === 0) return;
// onConflictDoUpdate with a no-op SET guarantees a RETURNING row regardless of
// whether the INSERT or the conflict path was taken, avoiding a read-committed
// race where onConflictDoNothing + subsequent SELECT could return nothing.
await executor.insert(narrators).values({ name, sortName }).onConflictDoUpdate({ target: narrators.name, set: { name } });
const [narrator] = await executor.select({ id: narrators.id }).from(narrators).where(eq(narrators.name, name)).limit(1);
if (narrator) {
await executor.insert(bookNarrators).values({ bookId, narratorId: narrator.id, displayOrder: i }).onConflictDoNothing();
const uniqueByName = new Map<string, { name: string; sortName: string | null }>();
for (const value of names) {
if (!uniqueByName.has(value.name)) {
uniqueByName.set(value.name, value);
}
}
const uniqueNames = [...uniqueByName.values()];
const narratorByName = new Map<string, { id: number }>();
const upsertedNarrators = await executor
.insert(narrators)
.values(uniqueNames)
.onConflictDoUpdate({
target: narrators.name,
set: { name: sql`excluded.name` },
})
.returning({ id: narrators.id, name: narrators.name });
for (const row of upsertedNarrators) {
narratorByName.set(row.name, { id: row.id });
}
const links = names.flatMap((value, index) => {
const match = narratorByName.get(value.name);
if (!match) return [];
return [{ bookId, narratorId: match.id, displayOrder: index }];
});
if (links.length > 0) {
await executor.insert(bookNarrators).values(links).onConflictDoNothing();
}
}
}
server/src/modules/opds/opds-book.service.ts
+32 -5
View File
@@ -1,5 +1,5 @@
import { ForbiddenException, Inject, Injectable } from '@nestjs/common';
import { SQL, and, count, eq, inArray, sql } from 'drizzle-orm';
import { SQL, and, count, eq, gte, inArray, lt, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db';
@@ -156,18 +156,45 @@ export class OpdsBookService {
}
async getRandomBooks(userId: number, count: number, isSuperuser = false): Promise<OpdsBookEntry[]> {
if (count <= 0) return [];
const accessibleIds = await this.getAccessibleLibraryIds(userId, isSuperuser);
if (accessibleIds.length === 0) return [];
const ids = await this.db
const baseFilter = and(inArray(books.libraryId, accessibleIds), eq(books.status, 'present'))!;
const [bounds] = await this.db
.select({
minId: sql<number | null>`min(${books.id})`,
maxId: sql<number | null>`max(${books.id})`,
})
.from(books)
.where(baseFilter);
if (bounds?.minId == null || bounds.maxId == null || bounds.minId > bounds.maxId) return [];
const range = bounds.maxId - bounds.minId + 1;
const anchorId = bounds.minId + Math.floor(Math.random() * range);
const firstPass = await this.db
.select({ id: books.id })
.from(books)
.where(and(inArray(books.libraryId, accessibleIds), eq(books.status, 'present')))
.orderBy(sql`random()`)
.where(and(baseFilter, gte(books.id, anchorId)))
.orderBy(books.id)
.limit(count);
const remaining = count - firstPass.length;
const secondPass =
remaining > 0
? await this.db
.select({ id: books.id })
.from(books)
.where(and(baseFilter, lt(books.id, anchorId)))
.orderBy(books.id)
.limit(remaining)
: [];
const ids = [...firstPass, ...secondPass].map((row) => row.id);
if (ids.length === 0) return [];
return this.fetchBookEntries(ids.map((r) => r.id));
return this.fetchBookEntries(ids);
}
async getDistinctAuthors(userId: number, isSuperuser = false): Promise<{ name: string; bookCount: number }[]> {
server/src/modules/opds/opds.controller.ts
+9
View File
@@ -18,6 +18,7 @@ import { ConfigService } from '@nestjs/config';
import type { FastifyReply } from 'fastify';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../common/constants/pagination.constants';
import { Public } from '../../common/decorators/public.decorator';
import { OPDS_MIME_ACQ, OPDS_MIME_NAV, OPDS_MIME_SEARCH, fileMimeType } from './opds-xml.helpers';
import { OpdsAuthGuard } from './opds-auth.guard';
@@ -41,6 +42,12 @@ export class OpdsController {
this.appDataPath = this.config.get<string>('storage.appDataPath')!;
}
private assertPaginationWindow(page: number, size: number): void {
if (!isOffsetWithinLimit((page - 1) * size)) {
throw new BadRequestException(`pagination window is too deep; (page - 1) * size must be <= ${MAX_OFFSET_ROWS}`);
}
}
@Get()
root(@OpdsUser() _user: OpdsRequestUser, @Res() reply: FastifyReply) {
const xml = this.opdsService.generateRootNavigation();
@@ -97,6 +104,7 @@ export class OpdsController {
) {
const clampedSize = Math.min(Math.max(size, 1), 100);
const clampedPage = Math.max(page, 1);
this.assertPaginationWindow(clampedPage, clampedSize);
const filters: Record<string, string | number> = {};
const libraryId = this.parseOptionalPositiveInt('libraryId', libraryIdStr);
@@ -148,6 +156,7 @@ export class OpdsController {
) {
const clampedSize = Math.min(Math.max(size, 1), 100);
const clampedPage = Math.max(page, 1);
this.assertPaginationWindow(clampedPage, clampedSize);
const { entries, total } = await this.opdsBookService.getRecentBooksPage(user.userId, clampedPage, clampedSize, user.isSuperuser);
const selfPath = `/api/v1/opds/recent?page=${clampedPage}&size=${clampedSize}`;
server/src/modules/scanner/scanner.repository.ts
+20 -2
View File
@@ -61,7 +61,14 @@ export class ScannerRepository {
// ── Books ──────────────────────────────────────────────────────────────────
async findBooksByLibraryFolder(libraryFolderId: number) {
return this.db.select().from(books).where(eq(books.libraryFolderId, libraryFolderId));
return this.db
.select({
id: books.id,
status: books.status,
folderPath: books.folderPath,
})
.from(books)
.where(eq(books.libraryFolderId, libraryFolderId));
}
async findPrimaryBookFilesByLibrary(libraryId: number) {
@@ -135,7 +142,18 @@ export class ScannerRepository {
// ── Book Files ─────────────────────────────────────────────────────────────
async findBookFilesByLibraryFolder(libraryFolderId: number) {
return this.db.select().from(bookFiles).where(eq(bookFiles.libraryFolderId, libraryFolderId));
return this.db
.select({
id: bookFiles.id,
bookId: bookFiles.bookId,
absolutePath: bookFiles.absolutePath,
ino: bookFiles.ino,
sizeBytes: bookFiles.sizeBytes,
mtime: bookFiles.mtime,
hash: bookFiles.hash,
})
.from(bookFiles)
.where(eq(bookFiles.libraryFolderId, libraryFolderId));
}
async findBookFileByHash(hash: string, libraryFolderId: number) {
server/src/modules/scanner/scanner.service.ts
+44 -59
View File
@@ -406,70 +406,53 @@ export class ScannerService implements OnApplicationBootstrap {
const startedAt = Date.now();
this.logger.log(`[${event}] [start] libraryId=${libraryId} jobId=${jobId} folderCount=${folders.length} - scan job started`);
type FolderWork = {
id: number;
libraryId: number;
path: string;
candidates: BookCandidate[];
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>;
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>;
};
const folderWork: FolderWork[] = [];
let totalCandidates = 0;
const allowed = allowedFormats.length > 0 ? new Set(allowedFormats) : null;
for (const folder of folders) {
let candidates: BookCandidate[] = [];
try {
candidates =
organizationMode === 'book_per_file'
? await findLooseFileCandidates(folder.path, excludePatterns, (msg) =>
this.logger.warn(
`[scanner.walk_candidates] [fail] libraryId=${libraryId} path="${folder.path.replace(/"/g, '\\"')}" error="${msg.replace(/"/g, '\\"')}" - candidate walk warning`,
),
)
: await findBookCandidates(folder.path, excludePatterns, (msg) =>
this.logger.warn(
`[scanner.walk_candidates] [fail] libraryId=${libraryId} path="${folder.path.replace(/"/g, '\\"')}" error="${msg.replace(/"/g, '\\"')}" - candidate walk warning`,
),
);
} catch (err) {
this.logger.warn(
`[${event}] [fail] libraryId=${libraryId} jobId=${jobId} path="${folder.path.replace(/"/g, '\\"')}" durationMs=${Date.now() - startedAt} errorClass=${err instanceof Error ? err.name : 'Error'} error="${(err instanceof Error ? err.message : String(err)).replace(/"/g, '\\"')}" - cannot walk folder`,
);
}
if (allowed) {
candidates = candidates
.map((c) => ({
...c,
files: c.files.filter((f) => {
const { role, format } = classifyFile(f.absolutePath);
return role !== 'content' || (format !== null && allowed.has(format));
}),
}))
.filter((c) => c.files.some((f) => classifyFile(f.absolutePath).role === 'content'));
}
const [knownBooks, knownFiles] = await Promise.all([
this.scannerRepo.findBooksByLibraryFolder(folder.id),
this.scannerRepo.findBookFilesByLibraryFolder(folder.id),
]);
folderWork.push({ ...folder, candidates, knownBooks, knownFiles });
totalCandidates += candidates.length;
}
this.scanJobStore.setTotal(libraryId, totalCandidates);
this.emitFromStore(libraryId, jobId, 'running');
const totals: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
try {
for (const { id: folderId, candidates, knownBooks, knownFiles } of folderWork) {
const counts = await this.scanFolderCandidates(folderId, libraryId, candidates, knownBooks, knownFiles, jobId, formatPriority);
this.scanJobStore.setTotal(libraryId, 0);
this.emitFromStore(libraryId, jobId, 'running');
for (const folder of folders) {
let candidates: BookCandidate[] = [];
try {
candidates =
organizationMode === 'book_per_file'
? await findLooseFileCandidates(folder.path, excludePatterns, (msg) =>
this.logger.warn(
`[scanner.walk_candidates] [fail] libraryId=${libraryId} path="${folder.path.replace(/"/g, '\\"')}" error="${msg.replace(/"/g, '\\"')}" - candidate walk warning`,
),
)
: await findBookCandidates(folder.path, excludePatterns, (msg) =>
this.logger.warn(
`[scanner.walk_candidates] [fail] libraryId=${libraryId} path="${folder.path.replace(/"/g, '\\"')}" error="${msg.replace(/"/g, '\\"')}" - candidate walk warning`,
),
);
} catch (err) {
this.logger.warn(
`[${event}] [fail] libraryId=${libraryId} jobId=${jobId} path="${folder.path.replace(/"/g, '\\"')}" durationMs=${Date.now() - startedAt} errorClass=${err instanceof Error ? err.name : 'Error'} error="${(err instanceof Error ? err.message : String(err)).replace(/"/g, '\\"')}" - cannot walk folder`,
);
}
if (allowed) {
candidates = candidates
.map((c) => ({
...c,
files: c.files.filter((f) => {
const { role, format } = classifyFile(f.absolutePath);
return role !== 'content' || (format !== null && allowed.has(format));
}),
}))
.filter((c) => c.files.some((f) => classifyFile(f.absolutePath).role === 'content'));
}
totalCandidates += candidates.length;
this.scanJobStore.setTotal(libraryId, totalCandidates);
const counts = await this.scanFolderCandidates(folder.id, libraryId, candidates, jobId, formatPriority);
totals.addedCount += counts.addedCount;
totals.updatedCount += counts.updatedCount;
totals.missingCount += counts.missingCount;
@@ -500,8 +483,6 @@ export class ScannerService implements OnApplicationBootstrap {
libraryFolderId: number,
libraryId: number,
candidates: BookCandidate[],
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>,
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>,
jobId: number,
formatPriority: string[],
): Promise<ScanCounts> {
@@ -512,6 +493,10 @@ export class ScannerService implements OnApplicationBootstrap {
);
try {
const counts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
const [knownBooks, knownFiles] = await Promise.all([
this.scannerRepo.findBooksByLibraryFolder(libraryFolderId),
this.scannerRepo.findBookFilesByLibraryFolder(libraryFolderId),
]);
const bookByFolderPath = new Map<string, { id: number; status: string; folderPath: string }>(
knownBooks.map((b) => [b.folderPath, { id: b.id, status: b.status, folderPath: b.folderPath }]),
server/src/modules/statistics/statistics.service.test.ts
+18
View File
@@ -212,4 +212,22 @@ describe('StatisticsService', () => {
expect(result.items).toEqual([{ name: 'Saga', count: 4 }]);
});
it('reuses cached metadata score distribution for identical scope', async () => {
const { service, repo } = makeService();
repo.metadataScoreDistribution.mockResolvedValue({
bins: [{ minScore: 0, count: 1 }],
unknownCount: 0,
totalCount: 1,
percentile25: 0,
percentile50: 0,
percentile75: 0,
percentile90: 0,
});
await service.getMetadataScoreDistribution(makeUser(), { libraryIds: [2, 1] });
await service.getMetadataScoreDistribution(makeUser(), { libraryIds: [1, 2] });
expect(repo.metadataScoreDistribution).toHaveBeenCalledTimes(1);
});
});
server/src/modules/statistics/statistics.service.ts
+230 -126
View File
@@ -35,6 +35,8 @@ const TOP_LIST_LIMIT = 25;
const METADATA_SCORE_BIN_COUNT = 10;
const OTHER_BUCKET_LABEL = 'Other';
const UNKNOWN_FORMAT_LABEL = 'UNKNOWN';
const STATISTICS_CACHE_TTL_MS = 30_000;
const STATISTICS_CACHE_MAX_ENTRIES = 200;
type MetadataCompletenessFieldKey =
| 'hasTitle'
@@ -75,6 +77,9 @@ const METADATA_COMPLETENESS_FIELDS: MetadataCompletenessFieldDefinition[] = [
@Injectable()
export class StatisticsService {
private readonly cache = new Map<string, { value: unknown; expiresAt: number }>();
private readonly inFlightCache = new Map<string, Promise<unknown>>();
constructor(private readonly repo: StatisticsRepository) {}
async getFormatDistribution(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<FormatDistributionItem>> {
@@ -90,108 +95,124 @@ export class StatisticsService {
}
async getBooksAddedOverTime(user: RequestUser, query: BooksOverTimeQueryDto): Promise<StatisticsResult<BooksAddedDataPoint>> {
const items = await this.repo.booksAddedOverTime(user.id, user.isSuperuser, query.libraryIds, query.granularity, query.range);
return { items, unknownCount: 0 };
return this.withStatisticsCache(
'books-added-over-time',
user,
query,
async () => {
const items = await this.repo.booksAddedOverTime(user.id, user.isSuperuser, query.libraryIds, query.granularity, query.range);
return { items, unknownCount: 0 };
},
{ granularity: query.granularity ?? 'monthly', range: query.range ?? 'all-time' },
);
}
async getMetadataScoreDistribution(user: RequestUser, query: StatisticsFilterQueryDto): Promise<MetadataScoreDistribution> {
const raw = await this.repo.metadataScoreDistribution(user.id, user.isSuperuser, query.libraryIds);
const byMin = new Map(raw.bins.map((b) => [b.minScore, b.count]));
const bins = Array.from({ length: METADATA_SCORE_BIN_COUNT }, (_, i) => {
const minScore = i * 10;
const maxScore = i === METADATA_SCORE_BIN_COUNT - 1 ? 100 : minScore + 9;
return this.withStatisticsCache('metadata-score-distribution', user, query, async () => {
const raw = await this.repo.metadataScoreDistribution(user.id, user.isSuperuser, query.libraryIds);
const byMin = new Map(raw.bins.map((b) => [b.minScore, b.count]));
const bins = Array.from({ length: METADATA_SCORE_BIN_COUNT }, (_, i) => {
const minScore = i * 10;
const maxScore = i === METADATA_SCORE_BIN_COUNT - 1 ? 100 : minScore + 9;
return {
minScore,
maxScore,
count: byMin.get(minScore) ?? 0,
};
});
return {
minScore,
maxScore,
count: byMin.get(minScore) ?? 0,
bins,
unknownCount: raw.unknownCount,
totalCount: raw.totalCount,
percentile25: raw.percentile25,
percentile50: raw.percentile50,
percentile75: raw.percentile75,
percentile90: raw.percentile90,
};
});
return {
bins,
unknownCount: raw.unknownCount,
totalCount: raw.totalCount,
percentile25: raw.percentile25,
percentile50: raw.percentile50,
percentile75: raw.percentile75,
percentile90: raw.percentile90,
};
}
async getLibraryMetadataCompleteness(
user: RequestUser,
query: StatisticsFilterQueryDto,
): Promise<StatisticsResult<LibraryMetadataCompletenessItem>> {
const rows = await this.repo.libraryMetadataCompleteness(user.id, user.isSuperuser, query.libraryIds);
return this.withStatisticsCache('library-metadata-completeness', user, query, async () => {
const rows = await this.repo.libraryMetadataCompleteness(user.id, user.isSuperuser, query.libraryIds);
const items: LibraryMetadataCompletenessItem[] = rows.flatMap((row) =>
METADATA_COMPLETENESS_FIELDS.map((f) => {
const presentCount = Number(row[f.key] ?? 0);
const totalCount = row.total ?? 0;
return {
libraryId: row.libraryId,
libraryName: row.libraryName,
field: f.field,
presentCount,
totalCount,
percent: totalCount > 0 ? Math.round((presentCount / totalCount) * 100) : 0,
};
}),
);
const items: LibraryMetadataCompletenessItem[] = rows.flatMap((row) =>
METADATA_COMPLETENESS_FIELDS.map((f) => {
const presentCount = Number(row[f.key] ?? 0);
const totalCount = row.total ?? 0;
return {
libraryId: row.libraryId,
libraryName: row.libraryName,
field: f.field,
presentCount,
totalCount,
percent: totalCount > 0 ? Math.round((presentCount / totalCount) * 100) : 0,
};
}),
);
return { items, unknownCount: 0 };
return { items, unknownCount: 0 };
});
}
async getFormatShareOverTime(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<FormatShareOverTimeItem>> {
const raw = await this.repo.formatShareOverTime(user.id, user.isSuperuser, query.libraryIds);
const totals = new Map<string, number>();
for (const row of raw) {
const format = this.normalizeFormatLabel(row.format);
totals.set(format, (totals.get(format) ?? 0) + row.count);
}
const topFormats = new Set(
[...totals.entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, STREAM_TOP_FORMATS)
.map(([f]) => f),
);
const grouped = new Map<string, FormatShareOverTimeItem>();
for (const row of raw) {
const normalizedFormat = this.normalizeFormatLabel(row.format);
const format = topFormats.has(normalizedFormat) ? normalizedFormat : 'OTHER';
const key = `${row.year}-${row.month}-${format}`;
const existing = grouped.get(key);
if (existing) {
existing.count += row.count;
continue;
return this.withStatisticsCache('format-share-over-time', user, query, async () => {
const raw = await this.repo.formatShareOverTime(user.id, user.isSuperuser, query.libraryIds);
const totals = new Map<string, number>();
for (const row of raw) {
const format = this.normalizeFormatLabel(row.format);
totals.set(format, (totals.get(format) ?? 0) + row.count);
}
grouped.set(key, { year: row.year, month: row.month, format, count: row.count });
}
const items = [...grouped.values()].sort((a, b) => a.year - b.year || a.month - b.month || a.format.localeCompare(b.format));
return { items, unknownCount: 0 };
const topFormats = new Set(
[...totals.entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, STREAM_TOP_FORMATS)
.map(([f]) => f),
);
const grouped = new Map<string, FormatShareOverTimeItem>();
for (const row of raw) {
const normalizedFormat = this.normalizeFormatLabel(row.format);
const format = topFormats.has(normalizedFormat) ? normalizedFormat : 'OTHER';
const key = `${row.year}-${row.month}-${format}`;
const existing = grouped.get(key);
if (existing) {
existing.count += row.count;
continue;
}
grouped.set(key, { year: row.year, month: row.month, format, count: row.count });
}
const items = [...grouped.values()].sort((a, b) => a.year - b.year || a.month - b.month || a.format.localeCompare(b.format));
return { items, unknownCount: 0 };
});
}
async getPageCountDistribution(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<PageCountDistributionItem>> {
const { items: raw, unknownCount } = await this.repo.pageCountDistributionByFormat(user.id, user.isSuperuser, query.libraryIds);
const items = raw.flatMap((row) =>
row.format
? [
{
format: row.format.toUpperCase(),
count: row.count,
min: row.min,
q1: Number(row.q1),
median: Number(row.median),
q3: Number(row.q3),
max: row.max,
},
]
: [],
);
return { items, unknownCount };
return this.withStatisticsCache('page-count-distribution', user, query, async () => {
const { items: raw, unknownCount } = await this.repo.pageCountDistributionByFormat(user.id, user.isSuperuser, query.libraryIds);
const items = raw.flatMap((row) =>
row.format
? [
{
format: row.format.toUpperCase(),
count: row.count,
min: row.min,
q1: Number(row.q1),
median: Number(row.median),
q3: Number(row.q3),
max: row.max,
},
]
: [],
);
return { items, unknownCount };
});
}
async getStorageByFormat(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<StorageByFormatItem>> {
@@ -206,8 +227,10 @@ export class StatisticsService {
}
async getPublicationYearTimeline(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<PublicationYearPoint>> {
const { items, unknownCount } = await this.repo.publicationYearTimeline(user.id, user.isSuperuser, query.libraryIds);
return { items, unknownCount };
return this.withStatisticsCache('publication-year-timeline', user, query, async () => {
const { items, unknownCount } = await this.repo.publicationYearTimeline(user.id, user.isSuperuser, query.libraryIds);
return { items, unknownCount };
});
}
async getTopAuthors(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<TopAuthorItem>> {
@@ -217,12 +240,14 @@ export class StatisticsService {
}
async getMetadataCompleteness(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<MetadataCompletenessItem>> {
const row = await this.repo.metadataCompleteness(user.id, user.isSuperuser, query.libraryIds);
const total = row?.total ?? 0;
const items = METADATA_COMPLETENESS_FIELDS.filter((fieldDef) => fieldDef.includeInOverall)
.map((fieldDef) => ({ field: fieldDef.field, presentCount: row?.[fieldDef.key] ?? 0, totalCount: total }))
.sort((a, b) => b.presentCount - a.presentCount);
return { items, unknownCount: 0 };
return this.withStatisticsCache('metadata-completeness', user, query, async () => {
const row = await this.repo.metadataCompleteness(user.id, user.isSuperuser, query.libraryIds);
const total = row?.total ?? 0;
const items = METADATA_COMPLETENESS_FIELDS.filter((fieldDef) => fieldDef.includeInOverall)
.map((fieldDef) => ({ field: fieldDef.field, presentCount: row?.[fieldDef.key] ?? 0, totalCount: total }))
.sort((a, b) => b.presentCount - a.presentCount);
return { items, unknownCount: 0 };
});
}
async getGenreDistribution(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<GenreDistributionItem>> {
@@ -232,52 +257,58 @@ export class StatisticsService {
}
async getMetadataFreshnessGauge(user: RequestUser, query: StatisticsFilterQueryDto): Promise<MetadataFreshnessGauge> {
const row = await this.repo.metadataFreshnessGauge(user.id, user.isSuperuser, query.libraryIds);
const totalBooks = row.totalBooks ?? 0;
const fresh30dCount = row.fresh30dCount ?? 0;
const stale31To90dCount = row.stale31To90dCount ?? 0;
const stale91To180dCount = row.stale91To180dCount ?? 0;
const staleOver180dCount = row.staleOver180dCount ?? 0;
const neverFetchedCount = row.neverFetchedCount ?? 0;
return this.withStatisticsCache('metadata-freshness-gauge', user, query, async () => {
const row = await this.repo.metadataFreshnessGauge(user.id, user.isSuperuser, query.libraryIds);
const totalBooks = row.totalBooks ?? 0;
const fresh30dCount = row.fresh30dCount ?? 0;
const stale31To90dCount = row.stale31To90dCount ?? 0;
const stale91To180dCount = row.stale91To180dCount ?? 0;
const staleOver180dCount = row.staleOver180dCount ?? 0;
const neverFetchedCount = row.neverFetchedCount ?? 0;
const weightedFreshness = fresh30dCount + stale31To90dCount * 0.7 + stale91To180dCount * 0.4 + staleOver180dCount * 0.15;
const freshnessScore = totalBooks > 0 ? Math.round((weightedFreshness / totalBooks) * 100) : 0;
const weightedFreshness = fresh30dCount + stale31To90dCount * 0.7 + stale91To180dCount * 0.4 + staleOver180dCount * 0.15;
const freshnessScore = totalBooks > 0 ? Math.round((weightedFreshness / totalBooks) * 100) : 0;
return {
totalBooks,
neverFetchedCount,
fresh30dCount,
stale31To90dCount,
stale91To180dCount,
staleOver180dCount,
freshnessScore,
};
return {
totalBooks,
neverFetchedCount,
fresh30dCount,
stale31To90dCount,
stale91To180dCount,
staleOver180dCount,
freshnessScore,
};
});
}
async getLibraryIntegrityGauge(user: RequestUser, query: StatisticsFilterQueryDto): Promise<LibraryIntegrityGauge> {
const row = await this.repo.libraryIntegrityGauge(user.id, user.isSuperuser, query.libraryIds);
const totalBooks = row.totalBooks ?? 0;
const presentCount = row.presentCount ?? 0;
const primaryFileCount = row.primaryFileCount ?? 0;
const metadataCount = row.metadataCount ?? 0;
return this.withStatisticsCache('library-integrity-gauge', user, query, async () => {
const row = await this.repo.libraryIntegrityGauge(user.id, user.isSuperuser, query.libraryIds);
const totalBooks = row.totalBooks ?? 0;
const presentCount = row.presentCount ?? 0;
const primaryFileCount = row.primaryFileCount ?? 0;
const metadataCount = row.metadataCount ?? 0;
const presentRatio = totalBooks > 0 ? presentCount / totalBooks : 0;
const primaryFileRatio = totalBooks > 0 ? primaryFileCount / totalBooks : 0;
const metadataRatio = totalBooks > 0 ? metadataCount / totalBooks : 0;
const integrityScore = Math.round(((presentRatio + primaryFileRatio + metadataRatio) / 3) * 100);
const presentRatio = totalBooks > 0 ? presentCount / totalBooks : 0;
const primaryFileRatio = totalBooks > 0 ? primaryFileCount / totalBooks : 0;
const metadataRatio = totalBooks > 0 ? metadataCount / totalBooks : 0;
const integrityScore = Math.round(((presentRatio + primaryFileRatio + metadataRatio) / 3) * 100);
return {
totalBooks,
presentCount,
primaryFileCount,
metadataCount,
integrityScore,
};
return {
totalBooks,
presentCount,
primaryFileCount,
metadataCount,
integrityScore,
};
});
}
async getAcquisitionLagScatter(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<AcquisitionLagPoint>> {
const { items, unknownCount } = await this.repo.acquisitionLagScatter(user.id, user.isSuperuser, query.libraryIds);
return { items, unknownCount };
return this.withStatisticsCache('acquisition-lag-scatter', user, query, async () => {
const { items, unknownCount } = await this.repo.acquisitionLagScatter(user.id, user.isSuperuser, query.libraryIds);
return { items, unknownCount };
});
}
private clipCountsToTopN<T extends { count: number }>(items: T[], createOtherItem: (count: number) => T, n = STATISTICS_TOP_N): T[] {
@@ -299,11 +330,13 @@ export class StatisticsService {
}
async getSummary(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsSummary> {
return this.repo.getSummary(user.id, user.isSuperuser, query.libraryIds);
return this.withStatisticsCache('summary', user, query, () => this.repo.getSummary(user.id, user.isSuperuser, query.libraryIds));
}
async getGenreCooccurrence(user: RequestUser, query: StatisticsFilterQueryDto): Promise<ChordDiagramData> {
return this.repo.getGenreCooccurrence(user.id, user.isSuperuser, query.libraryIds);
return this.withStatisticsCache('genre-cooccurrence', user, query, () =>
this.repo.getGenreCooccurrence(user.id, user.isSuperuser, query.libraryIds),
);
}
async getLargestBooks(user: RequestUser, query: StatisticsFilterQueryDto): Promise<StatisticsResult<LargestBookItem>> {
@@ -317,4 +350,75 @@ export class StatisticsService {
const items = raw.flatMap((r) => (r.name ? [{ name: r.name, count: r.count }] : []));
return { items, unknownCount: 0 };
}
private withStatisticsCache<T>(
endpoint: string,
user: RequestUser,
query: { libraryIds?: number[] },
load: () => Promise<T>,
extraScope: Record<string, string | number | boolean> = {},
): Promise<T> {
const key = this.buildCacheKey(endpoint, user, query.libraryIds, extraScope);
return this.withCache(key, load);
}
private buildCacheKey(
endpoint: string,
user: RequestUser,
libraryIds: number[] | undefined,
extraScope: Record<string, string | number | boolean>,
): string {
const normalizedLibraries = [...new Set(libraryIds ?? [])].sort((a, b) => a - b);
return JSON.stringify({
endpoint,
userId: user.id,
isSuperuser: user.isSuperuser,
libraryIds: normalizedLibraries,
...extraScope,
});
}
private async withCache<T>(key: string, load: () => Promise<T>): Promise<T> {
const now = Date.now();
const cached = this.cache.get(key);
if (cached && cached.expiresAt > now) {
return cached.value as T;
}
if (cached) {
this.cache.delete(key);
}
const inFlight = this.inFlightCache.get(key);
if (inFlight) {
return inFlight as Promise<T>;
}
const pending = load()
.then((value) => {
this.cache.set(key, { value, expiresAt: Date.now() + STATISTICS_CACHE_TTL_MS });
this.pruneCache();
return value;
})
.finally(() => {
this.inFlightCache.delete(key);
});
this.inFlightCache.set(key, pending as Promise<unknown>);
return pending;
}
private pruneCache(): void {
const now = Date.now();
for (const [key, entry] of this.cache) {
if (entry.expiresAt <= now) {
this.cache.delete(key);
}
}
while (this.cache.size > STATISTICS_CACHE_MAX_ENTRIES) {
const oldestKey = this.cache.keys().next().value as string | undefined;
if (!oldestKey) break;
this.cache.delete(oldestKey);
}
}
}