mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
feat: pgvector ANN + rescore recommendation engine with admin rebuild UI
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
<script setup lang="ts">
|
||||
import { watch } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
|
||||
import { bookCoverStyle } from '@/features/book/lib/book-cover'
|
||||
import { useRecommendations } from '@/features/book/composables/useRecommendations'
|
||||
import { useCoverVersions } from '@/features/book/composables/useCoverVersions'
|
||||
|
||||
const props = defineProps<{ bookId: number }>()
|
||||
const router = useRouter()
|
||||
const { recommendations, loading, fetch } = useRecommendations()
|
||||
const { coverUrl } = useCoverVersions()
|
||||
|
||||
watch(() => props.bookId, (id) => fetch(id), { immediate: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div v-if="loading || recommendations.length > 0" class="mt-8 pt-6 border-t border-border">
|
||||
<p class="text-[11px] font-semibold uppercase tracking-widest text-muted-foreground mb-4">More Like This</p>
|
||||
|
||||
<div v-if="loading" class="flex gap-3 overflow-x-auto pb-2">
|
||||
<div v-for="i in 10" :key="i" class="w-24 shrink-0">
|
||||
<div class="w-full rounded-sm bg-muted animate-pulse" style="aspect-ratio: 2/3" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else class="flex gap-6 overflow-x-auto pb-2">
|
||||
<button
|
||||
v-for="rec in recommendations"
|
||||
:key="rec.book.id"
|
||||
class="w-30 shrink-0 text-left group"
|
||||
@click="router.push({ name: 'book-detail', params: { bookId: rec.book.id } })"
|
||||
>
|
||||
<div
|
||||
class="w-full rounded-sm overflow-hidden shadow-sm group-hover:shadow-md group-hover:scale-[1.02] transition-all duration-150"
|
||||
style="aspect-ratio: 2/3"
|
||||
:style="bookCoverStyle(rec.book.title ?? String(rec.book.id))"
|
||||
>
|
||||
<img
|
||||
:src="coverUrl(rec.book.id, 'thumbnail')"
|
||||
:alt="rec.book.title ?? ''"
|
||||
class="w-full h-full object-cover"
|
||||
loading="lazy"
|
||||
@error="(e) => ((e.target as HTMLImageElement).style.display = 'none')"
|
||||
/>
|
||||
</div>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -6,6 +6,7 @@ import { DialogClose, DialogContent, DialogOverlay, DialogPortal, DialogRoot } f
|
||||
import { bookCoverStyle } from '@/features/book/lib/book-cover'
|
||||
import { useCoverVersions } from '@/features/book/composables/useCoverVersions'
|
||||
import type { BookDetail } from '@projectx/types'
|
||||
import RecommendedBooksRow from '@/features/book/components/detail/RecommendedBooksRow.vue'
|
||||
|
||||
const props = defineProps<{ book: BookDetail }>()
|
||||
const router = useRouter()
|
||||
@@ -201,7 +202,7 @@ function downloadFile() {
|
||||
<div v-if="book.description">
|
||||
<div
|
||||
class="text-sm leading-relaxed text-foreground/80 transition-all"
|
||||
:class="descriptionExpanded ? '' : 'line-clamp-6'"
|
||||
:class="descriptionExpanded ? '' : 'line-clamp-3'"
|
||||
v-html="book.description"
|
||||
/>
|
||||
<button
|
||||
@@ -216,6 +217,8 @@ function downloadFile() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<RecommendedBooksRow :book-id="book.id" />
|
||||
|
||||
<!-- Cover lightbox -->
|
||||
<DialogRoot :open="coverLightboxOpen" @update:open="coverLightboxOpen = $event">
|
||||
<DialogPortal>
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { ref } from 'vue'
|
||||
|
||||
import type { BookRecommendation } from '@projectx/types'
|
||||
import { api } from '@/lib/api'
|
||||
|
||||
export function useRecommendations() {
|
||||
const recommendations = ref<BookRecommendation[]>([])
|
||||
const loading = ref(false)
|
||||
|
||||
async function fetch(bookId: number) {
|
||||
loading.value = true
|
||||
recommendations.value = []
|
||||
try {
|
||||
const res = await api(`/api/books/${bookId}/recommendations`)
|
||||
if (!res.ok) return
|
||||
recommendations.value = await res.json()
|
||||
} catch {
|
||||
// recommendations are non-critical, fail silently
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
return { recommendations, loading, fetch }
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import { Check, RefreshCw, Sparkles } from 'lucide-vue-next'
|
||||
|
||||
import { api } from '@/lib/api'
|
||||
|
||||
const running = ref(false)
|
||||
const queued = ref<number | null>(null)
|
||||
const error = ref<string | null>(null)
|
||||
|
||||
async function rebuildEmbeddings() {
|
||||
running.value = true
|
||||
queued.value = null
|
||||
error.value = null
|
||||
try {
|
||||
const res = await api('/api/books/embed-all', { method: 'POST' })
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||
const data: { queued: number } = await res.json()
|
||||
queued.value = data.queued
|
||||
} catch (e) {
|
||||
error.value = e instanceof Error ? e.message : 'Failed'
|
||||
} finally {
|
||||
running.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="px-5 py-6 sm:px-10 sm:py-8 max-w-3xl mx-auto">
|
||||
<div class="mb-8">
|
||||
<h2 class="font-serif font-semibold text-foreground text-2xl tracking-tight">Maintenance</h2>
|
||||
<p class="mt-1 text-sm text-muted-foreground">System maintenance and data operations.</p>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p class="text-xs font-semibold text-muted-foreground uppercase tracking-wider mb-3">Recommendations</p>
|
||||
<div class="border border-border rounded-lg bg-card px-5 py-5">
|
||||
<div class="flex items-start justify-between gap-6">
|
||||
<div class="flex items-start gap-3">
|
||||
<div class="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg bg-primary/10">
|
||||
<Sparkles :size="16" class="text-primary" />
|
||||
</div>
|
||||
<div>
|
||||
<p class="text-sm font-medium text-foreground">Rebuild recommendation embeddings</p>
|
||||
<p class="text-xs text-muted-foreground mt-1 leading-relaxed max-w-sm">
|
||||
Generates vector embeddings for all books. Run this after a large import or if recommendations seem off. Processes in the background.
|
||||
</p>
|
||||
<p v-if="queued !== null" class="flex items-center gap-1.5 text-xs text-green-600 dark:text-green-400 mt-2">
|
||||
<Check :size="12" />
|
||||
{{ queued }} books queued for processing
|
||||
</p>
|
||||
<p v-if="error" class="text-xs text-destructive mt-2">{{ error }}</p>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
class="flex shrink-0 items-center gap-2 h-8 px-3 rounded-md border border-border bg-background text-sm text-foreground hover:bg-muted transition-colors disabled:opacity-50"
|
||||
:disabled="running"
|
||||
@click="rebuildEmbeddings"
|
||||
>
|
||||
<RefreshCw :size="13" :class="running ? 'animate-spin' : ''" />
|
||||
{{ running ? 'Running...' : 'Run' }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
LogIn,
|
||||
Rss,
|
||||
Tablet,
|
||||
Wrench,
|
||||
} from 'lucide-vue-next'
|
||||
import { useSettingsDrawer } from '@/composables/useSettingsDrawer'
|
||||
import { usePermissions } from '@/features/auth/composables/usePermissions'
|
||||
@@ -34,6 +35,7 @@ import PermissionsPage from '@/features/admin/PermissionsPage.vue'
|
||||
import OidcSettings from './OidcSettings.vue'
|
||||
import OpdsSettings from './OpdsSettings.vue'
|
||||
import KoboSettings from './KoboSettings.vue'
|
||||
import MaintenanceSettings from './MaintenanceSettings.vue'
|
||||
|
||||
const { isOpen, close } = useSettingsDrawer()
|
||||
const { isSuperuser, userPermissions } = usePermissions()
|
||||
@@ -54,6 +56,7 @@ type SectionId =
|
||||
| 'roles'
|
||||
| 'permissions'
|
||||
| 'oidc'
|
||||
| 'maintenance'
|
||||
|
||||
const navGroups = computed(() => {
|
||||
const perms = userPermissions.value
|
||||
@@ -88,6 +91,7 @@ const navGroups = computed(() => {
|
||||
}
|
||||
if (su || perms.includes('manage_app_settings')) {
|
||||
adminItems.push({ id: 'oidc', label: 'OIDC / SSO', icon: LogIn, component: OidcSettings })
|
||||
adminItems.push({ id: 'maintenance', label: 'Maintenance', icon: Wrench, component: MaintenanceSettings })
|
||||
}
|
||||
if (adminItems.length) {
|
||||
groups.push({ label: 'Administration', items: adminItems })
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ name: projectx-dev
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
image: pgvector/pgvector:pg16
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-projectx}
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm"; -- for fuzzy text search
|
||||
CREATE EXTENSION IF NOT EXISTS "vector"; -- for pgvector ANN search
|
||||
|
||||
@@ -66,3 +66,8 @@ export type BooksPage = {
|
||||
page: number;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export type BookRecommendation = {
|
||||
book: BookCard;
|
||||
score: number;
|
||||
};
|
||||
|
||||
@@ -26,6 +26,7 @@ import { PathModule } from './modules/path/path.module';
|
||||
import { MetadataFetchModule } from './modules/metadata-fetch/metadata-fetch.module';
|
||||
import { MetadataModule } from './modules/metadata/metadata.module';
|
||||
import { ReaderPreferencesModule } from './modules/reader-preferences/reader-preferences.module';
|
||||
import { RecommendationModule } from './modules/recommendation/recommendation.module';
|
||||
import { RoleModule } from './modules/role/role.module';
|
||||
import { ScannerModule } from './modules/scanner/scanner.module';
|
||||
import { SeedModule } from './modules/seed/seed.module';
|
||||
@@ -55,6 +56,7 @@ import { UserModule } from './modules/user/user.module';
|
||||
ScannerModule,
|
||||
MetadataModule,
|
||||
MetadataFetchModule,
|
||||
RecommendationModule,
|
||||
KoboModule,
|
||||
OpdsModule,
|
||||
BookmarkModule,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE "book_metadata" ADD COLUMN "embedding" vector(256);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,9 +5,16 @@
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1771545632263,
|
||||
"when": 1771614528351,
|
||||
"tag": "0000_baseline",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "7",
|
||||
"when": 1771614528351,
|
||||
"tag": "0001_add_embedding_column",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
import { integer, pgTable, primaryKey, real, serial, text, timestamp, varchar } from 'drizzle-orm/pg-core';
|
||||
import { customType, integer, pgTable, primaryKey, real, serial, text, timestamp, varchar } from 'drizzle-orm/pg-core';
|
||||
|
||||
const embedding256 = customType<{ data: number[]; driverData: string }>({
|
||||
dataType: () => 'vector(256)',
|
||||
toDriver: (v) => `[${v.join(',')}]`,
|
||||
fromDriver: (v) => v.slice(1, -1).split(',').map(Number),
|
||||
});
|
||||
|
||||
import { books } from './books';
|
||||
|
||||
@@ -24,6 +30,7 @@ export const bookMetadata = pgTable('book_metadata', {
|
||||
amazonId: varchar('amazon_id', { length: 20 }),
|
||||
hardcoverId: varchar('hardcover_id', { length: 50 }),
|
||||
openLibraryId: varchar('open_library_id', { length: 50 }),
|
||||
embedding: embedding256('embedding'),
|
||||
updatedAt: timestamp('updated_at').defaultNow().notNull(),
|
||||
});
|
||||
|
||||
|
||||
@@ -32,6 +32,12 @@ import type { BookQuery } from '@projectx/types';
|
||||
export class BookController {
|
||||
constructor(private readonly bookService: BookService) {}
|
||||
|
||||
@Post('embed-all')
|
||||
@RequirePermission('manage_app_settings')
|
||||
embedAll() {
|
||||
return this.bookService.embedAll();
|
||||
}
|
||||
|
||||
@Delete()
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@RequirePermission('library_delete_books')
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
|
||||
import { EmbeddingModule } from '../embedding/embedding.module';
|
||||
import { LibraryModule } from '../library/library.module';
|
||||
import { MetadataModule } from '../metadata/metadata.module';
|
||||
import { BookQueryBuilder } from './book-query-builder.service';
|
||||
@@ -8,7 +9,7 @@ import { BookRepository } from './book.repository';
|
||||
import { BookService } from './book.service';
|
||||
|
||||
@Module({
|
||||
imports: [forwardRef(() => LibraryModule), MetadataModule],
|
||||
imports: [forwardRef(() => LibraryModule), MetadataModule, EmbeddingModule],
|
||||
controllers: [BookController],
|
||||
providers: [BookService, BookRepository, BookQueryBuilder],
|
||||
exports: [BookService, BookRepository, BookQueryBuilder],
|
||||
|
||||
@@ -209,6 +209,11 @@ export class BookRepository {
|
||||
return this.db.select({ id: books.id, libraryId: books.libraryId }).from(books).where(inArray(books.id, bookIds));
|
||||
}
|
||||
|
||||
async findAllIds(): Promise<number[]> {
|
||||
const rows = await this.db.select({ id: books.id }).from(books);
|
||||
return rows.map((r) => r.id);
|
||||
}
|
||||
|
||||
async deleteByIds(bookIds: number[]): Promise<void> {
|
||||
await this.db.delete(books).where(inArray(books.id, bookIds));
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Injectable, Logger, NotFoundException, Optional } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { access, readdir, rm, stat } from 'fs/promises';
|
||||
import { basename, join } from 'path';
|
||||
@@ -7,6 +7,7 @@ import { MetadataProviderKey } from '@projectx/types';
|
||||
import type { BookQuery, BooksPage } from '@projectx/types';
|
||||
import { assembleBookCards } from './utils/assemble-book-cards';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { BookEmbedderService } from '../embedding/book-embedder.service';
|
||||
import { MetadataService } from '../metadata/metadata.service';
|
||||
import { LibraryService } from '../library/library.service';
|
||||
import { BookQueryBuilder } from './book-query-builder.service';
|
||||
@@ -26,6 +27,7 @@ export class BookService {
|
||||
private readonly queryBuilder: BookQueryBuilder,
|
||||
private readonly metadataService: MetadataService,
|
||||
private readonly config: ConfigService,
|
||||
@Optional() private readonly embedder: BookEmbedderService,
|
||||
) {
|
||||
this.booksPath = this.config.get<string>('storage.booksPath')!;
|
||||
}
|
||||
@@ -173,9 +175,28 @@ export class BookService {
|
||||
await this.metadataService.replaceTags(id, dto.tags);
|
||||
}
|
||||
|
||||
this.embedder?.embedBook(id).catch((err: Error) => this.logger.warn(`Embedding failed for book ${id}: ${err.message}`));
|
||||
return this.getDetail(id, user);
|
||||
}
|
||||
|
||||
async embedAll(): Promise<{ queued: number }> {
|
||||
const bookIds = await this.bookRepo.findAllIds();
|
||||
this.runEmbeddings(bookIds).catch((err: Error) => this.logger.error(`Embed-all failed: ${err.message}`));
|
||||
return { queued: bookIds.length };
|
||||
}
|
||||
|
||||
private async runEmbeddings(bookIds: number[]): Promise<void> {
|
||||
const BATCH = 10;
|
||||
for (let i = 0; i < bookIds.length; i += BATCH) {
|
||||
await Promise.all(
|
||||
bookIds
|
||||
.slice(i, i + BATCH)
|
||||
.map((id) => this.embedder?.embedBook(id).catch((err: Error) => this.logger.warn(`Failed to embed book ${id}: ${err.message}`))),
|
||||
);
|
||||
}
|
||||
this.logger.log(`Embeddings complete: ${bookIds.length} books processed`);
|
||||
}
|
||||
|
||||
async getProgress(userId: number, fileId: number, user: RequestUser) {
|
||||
await this.verifyFileAccess(fileId, user);
|
||||
return this.bookRepo.findProgress(userId, fileId);
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
import { authors, bookAuthors, bookGenres, bookMetadata, bookTags, genres, tags } from '../../db/schema';
|
||||
|
||||
type Db = NodePgDatabase<typeof schema>;
|
||||
|
||||
const DIMS = 256;
|
||||
|
||||
const STOPWORDS = new Set([
|
||||
'the',
|
||||
'a',
|
||||
'an',
|
||||
'and',
|
||||
'or',
|
||||
'of',
|
||||
'in',
|
||||
'to',
|
||||
'for',
|
||||
'is',
|
||||
'it',
|
||||
'on',
|
||||
'at',
|
||||
'by',
|
||||
'with',
|
||||
'as',
|
||||
'be',
|
||||
'this',
|
||||
'that',
|
||||
'are',
|
||||
'was',
|
||||
'were',
|
||||
'has',
|
||||
'have',
|
||||
'had',
|
||||
'not',
|
||||
'but',
|
||||
'from',
|
||||
'can',
|
||||
'all',
|
||||
'so',
|
||||
'if',
|
||||
'no',
|
||||
'will',
|
||||
'would',
|
||||
'could',
|
||||
'should',
|
||||
'may',
|
||||
'might',
|
||||
'do',
|
||||
'does',
|
||||
'did',
|
||||
'been',
|
||||
'being',
|
||||
'into',
|
||||
'out',
|
||||
'up',
|
||||
'one',
|
||||
'two',
|
||||
'more',
|
||||
'most',
|
||||
'also',
|
||||
'just',
|
||||
'about',
|
||||
'after',
|
||||
'its',
|
||||
]);
|
||||
|
||||
@Injectable()
|
||||
export class BookEmbedderService {
|
||||
private readonly logger = new Logger(BookEmbedderService.name);
|
||||
|
||||
constructor(@Inject(DB) private readonly db: Db) {}
|
||||
|
||||
async embedBook(bookId: number): Promise<number[] | null> {
|
||||
const [meta] = await this.db
|
||||
.select({
|
||||
title: bookMetadata.title,
|
||||
seriesName: bookMetadata.seriesName,
|
||||
publisher: bookMetadata.publisher,
|
||||
description: bookMetadata.description,
|
||||
})
|
||||
.from(bookMetadata)
|
||||
.where(eq(bookMetadata.bookId, bookId))
|
||||
.limit(1);
|
||||
|
||||
if (!meta) return null;
|
||||
|
||||
const [authorRows, genreRows, tagRows] = await Promise.all([
|
||||
this.db
|
||||
.select({ name: authors.name })
|
||||
.from(bookAuthors)
|
||||
.innerJoin(authors, eq(authors.id, bookAuthors.authorId))
|
||||
.where(eq(bookAuthors.bookId, bookId)),
|
||||
this.db
|
||||
.select({ name: genres.name })
|
||||
.from(bookGenres)
|
||||
.innerJoin(genres, eq(genres.id, bookGenres.genreId))
|
||||
.where(eq(bookGenres.bookId, bookId)),
|
||||
this.db.select({ name: tags.name }).from(bookTags).innerJoin(tags, eq(tags.id, bookTags.tagId)).where(eq(bookTags.bookId, bookId)),
|
||||
]);
|
||||
|
||||
const embedding = this.buildVector({
|
||||
title: meta.title,
|
||||
seriesName: meta.seriesName,
|
||||
publisher: meta.publisher,
|
||||
description: meta.description,
|
||||
authors: authorRows.map((r) => r.name),
|
||||
genres: genreRows.map((r) => r.name),
|
||||
tags: tagRows.map((r) => r.name),
|
||||
});
|
||||
|
||||
await this.db.update(bookMetadata).set({ embedding }).where(eq(bookMetadata.bookId, bookId));
|
||||
|
||||
this.logger.debug(`Embedded book ${bookId}`);
|
||||
return embedding;
|
||||
}
|
||||
|
||||
buildVector(params: {
|
||||
title: string | null;
|
||||
authors: string[];
|
||||
genres: string[];
|
||||
tags: string[];
|
||||
seriesName: string | null;
|
||||
publisher: string | null;
|
||||
description: string | null;
|
||||
}): number[] {
|
||||
const acc = new Float64Array(DIMS);
|
||||
|
||||
const addExact = (text: string, weight: number) => {
|
||||
const key = text.toLowerCase().trim();
|
||||
if (key.length > 0) acc[this.fnv32(key) % DIMS] += weight;
|
||||
};
|
||||
|
||||
const addTokens = (text: string, weight: number) => {
|
||||
for (const token of this.tokenize(text)) {
|
||||
acc[this.fnv32(token) % DIMS] += weight;
|
||||
}
|
||||
};
|
||||
|
||||
if (params.seriesName) addExact(params.seriesName, 3.0);
|
||||
for (const author of params.authors) addExact(author, 1.0);
|
||||
for (const genre of params.genres) addExact(genre, 4.0);
|
||||
for (const tag of params.tags) addExact(tag, 3.5);
|
||||
if (params.title) addTokens(params.title, 3.0);
|
||||
if (params.publisher) addExact(params.publisher, 2.0);
|
||||
if (params.description) {
|
||||
addTokens(params.description.split(/\s+/).slice(0, 100).join(' '), 1.0);
|
||||
}
|
||||
|
||||
return this.l2normalize(Array.from(acc));
|
||||
}
|
||||
|
||||
private tokenize(text: string): string[] {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter((t) => t.length >= 2 && !STOPWORDS.has(t));
|
||||
}
|
||||
|
||||
private fnv32(s: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
hash ^= s.charCodeAt(i);
|
||||
hash = Math.imul(hash, 0x01000193) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
private l2normalize(vec: number[]): number[] {
|
||||
const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
|
||||
if (norm === 0) return vec;
|
||||
return vec.map((v) => v / norm);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { BookEmbedderService } from './book-embedder.service';
|
||||
|
||||
@Module({
|
||||
providers: [BookEmbedderService],
|
||||
exports: [BookEmbedderService],
|
||||
})
|
||||
export class EmbeddingModule {}
|
||||
@@ -1,8 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { EmbeddingModule } from '../embedding/embedding.module';
|
||||
import { MetadataService } from './metadata.service';
|
||||
|
||||
@Module({
|
||||
imports: [EmbeddingModule],
|
||||
providers: [MetadataService],
|
||||
exports: [MetadataService],
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { and, eq, isNull } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
@@ -7,6 +7,7 @@ import { join } from 'path';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
import { BookEmbedderService } from '../embedding/book-embedder.service';
|
||||
import { authors, bookAuthors, bookGenres, bookMetadata, bookTags, genres, tags } from '../../db/schema';
|
||||
import { extractCb7Metadata, extractCbrMetadata, extractCbzMetadata } from './lib/cbz-metadata';
|
||||
import { extractAndSaveCover, generateThumbnail } from './lib/cover';
|
||||
@@ -25,6 +26,7 @@ export class MetadataService {
|
||||
constructor(
|
||||
@Inject(DB) private readonly db: Db,
|
||||
private readonly config: ConfigService,
|
||||
@Optional() private readonly embedder: BookEmbedderService,
|
||||
) {
|
||||
this.booksPath = this.config.get<string>('storage.booksPath')!;
|
||||
}
|
||||
@@ -170,6 +172,7 @@ export class MetadataService {
|
||||
await this.replaceGenres(bookId, parsed.tags);
|
||||
|
||||
this.logger.debug(`Metadata saved for book ${bookId}: "${parsed.title}"`);
|
||||
this.embedder?.embedBook(bookId).catch((err: Error) => this.logger.warn(`Embedding failed for book ${bookId}: ${err.message}`));
|
||||
}
|
||||
|
||||
// ── Cover ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Controller, Get, Param, ParseIntPipe } from '@nestjs/common';
|
||||
|
||||
import { CurrentUser } from '../../common/decorators/current-user.decorator';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { RecommendationService } from './recommendation.service';
|
||||
|
||||
@Controller('books')
|
||||
export class RecommendationController {
|
||||
constructor(private readonly recommendationService: RecommendationService) {}
|
||||
|
||||
@Get(':id/recommendations')
|
||||
getRecommendations(@Param('id', ParseIntPipe) id: number, @CurrentUser() user: RequestUser) {
|
||||
return this.recommendationService.getRecommendations(id, user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { BookModule } from '../book/book.module';
|
||||
import { EmbeddingModule } from '../embedding/embedding.module';
|
||||
import { LibraryModule } from '../library/library.module';
|
||||
import { RecommendationController } from './recommendation.controller';
|
||||
import { RecommendationRepository } from './recommendation.repository';
|
||||
import { RecommendationService } from './recommendation.service';
|
||||
|
||||
@Module({
|
||||
imports: [BookModule, LibraryModule, EmbeddingModule],
|
||||
controllers: [RecommendationController],
|
||||
providers: [RecommendationService, RecommendationRepository],
|
||||
})
|
||||
export class RecommendationModule {}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { eq, inArray, ne, and, isNotNull, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
import { authors, bookAuthors, bookGenres, bookMetadata, bookTags, books, genres, tags } from '../../db/schema';
|
||||
|
||||
type Db = NodePgDatabase<typeof schema>;
|
||||
|
||||
export interface AnnCandidate {
|
||||
bookId: number;
|
||||
cosineSim: number;
|
||||
seriesName: string | null;
|
||||
rating: number | null;
|
||||
}
|
||||
|
||||
export interface CandidateMetadata {
|
||||
bookId: number;
|
||||
authorNames: string[];
|
||||
genreTagNames: string[];
|
||||
}
|
||||
|
||||
export interface TargetBookData {
|
||||
embedding: number[] | null;
|
||||
seriesName: string | null;
|
||||
rating: number | null;
|
||||
authorNames: string[];
|
||||
genreTagNames: string[];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class RecommendationRepository {
|
||||
constructor(@Inject(DB) private readonly db: Db) {}
|
||||
|
||||
async getTargetBookData(bookId: number): Promise<TargetBookData | null> {
|
||||
const [meta] = await this.db
|
||||
.select({
|
||||
embedding: bookMetadata.embedding,
|
||||
seriesName: bookMetadata.seriesName,
|
||||
rating: bookMetadata.rating,
|
||||
})
|
||||
.from(bookMetadata)
|
||||
.where(eq(bookMetadata.bookId, bookId))
|
||||
.limit(1);
|
||||
|
||||
if (!meta) return null;
|
||||
|
||||
const [authorRows, genreRows, tagRows] = await Promise.all([
|
||||
this.db
|
||||
.select({ name: authors.name })
|
||||
.from(bookAuthors)
|
||||
.innerJoin(authors, eq(authors.id, bookAuthors.authorId))
|
||||
.where(eq(bookAuthors.bookId, bookId)),
|
||||
this.db
|
||||
.select({ name: genres.name })
|
||||
.from(bookGenres)
|
||||
.innerJoin(genres, eq(genres.id, bookGenres.genreId))
|
||||
.where(eq(bookGenres.bookId, bookId)),
|
||||
this.db.select({ name: tags.name }).from(bookTags).innerJoin(tags, eq(tags.id, bookTags.tagId)).where(eq(bookTags.bookId, bookId)),
|
||||
]);
|
||||
|
||||
return {
|
||||
embedding: meta.embedding,
|
||||
seriesName: meta.seriesName,
|
||||
rating: meta.rating,
|
||||
authorNames: authorRows.map((r) => r.name),
|
||||
genreTagNames: [...genreRows.map((r) => r.name), ...tagRows.map((r) => r.name)],
|
||||
};
|
||||
}
|
||||
|
||||
async findAnnCandidates(embedding: number[], targetBookId: number, libraryIds: number[]): Promise<AnnCandidate[]> {
|
||||
if (libraryIds.length === 0) return [];
|
||||
|
||||
const vecStr = `[${embedding.join(',')}]`;
|
||||
|
||||
return this.db
|
||||
.select({
|
||||
bookId: bookMetadata.bookId,
|
||||
cosineSim: sql<number>`(1 - (${bookMetadata.embedding} <=> ${vecStr}::vector))::float`,
|
||||
seriesName: bookMetadata.seriesName,
|
||||
rating: bookMetadata.rating,
|
||||
})
|
||||
.from(bookMetadata)
|
||||
.innerJoin(books, eq(books.id, bookMetadata.bookId))
|
||||
.where(and(inArray(books.libraryId, libraryIds), ne(bookMetadata.bookId, targetBookId), isNotNull(bookMetadata.embedding)))
|
||||
.orderBy(sql`${bookMetadata.embedding} <=> ${vecStr}::vector`)
|
||||
.limit(100);
|
||||
}
|
||||
|
||||
async getCandidateMetadata(bookIds: number[]): Promise<CandidateMetadata[]> {
|
||||
if (bookIds.length === 0) return [];
|
||||
|
||||
const [authorRows, genreRows, tagRows] = await Promise.all([
|
||||
this.db
|
||||
.select({ bookId: bookAuthors.bookId, name: authors.name })
|
||||
.from(bookAuthors)
|
||||
.innerJoin(authors, eq(authors.id, bookAuthors.authorId))
|
||||
.where(inArray(bookAuthors.bookId, bookIds)),
|
||||
this.db
|
||||
.select({ bookId: bookGenres.bookId, name: genres.name })
|
||||
.from(bookGenres)
|
||||
.innerJoin(genres, eq(genres.id, bookGenres.genreId))
|
||||
.where(inArray(bookGenres.bookId, bookIds)),
|
||||
this.db
|
||||
.select({ bookId: bookTags.bookId, name: tags.name })
|
||||
.from(bookTags)
|
||||
.innerJoin(tags, eq(tags.id, bookTags.tagId))
|
||||
.where(inArray(bookTags.bookId, bookIds)),
|
||||
]);
|
||||
|
||||
const authorsByBook = new Map<number, string[]>();
|
||||
for (const r of authorRows) {
|
||||
const list = authorsByBook.get(r.bookId) ?? [];
|
||||
list.push(r.name);
|
||||
authorsByBook.set(r.bookId, list);
|
||||
}
|
||||
|
||||
const genreTagsByBook = new Map<number, string[]>();
|
||||
for (const r of [...genreRows, ...tagRows]) {
|
||||
const list = genreTagsByBook.get(r.bookId) ?? [];
|
||||
list.push(r.name);
|
||||
genreTagsByBook.set(r.bookId, list);
|
||||
}
|
||||
|
||||
return bookIds.map((id) => ({
|
||||
bookId: id,
|
||||
authorNames: authorsByBook.get(id) ?? [],
|
||||
genreTagNames: genreTagsByBook.get(id) ?? [],
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { inArray } from 'drizzle-orm';
|
||||
|
||||
import type { BookRecommendation } from '@projectx/types';
|
||||
import { assembleBookCards } from '../book/utils/assemble-book-cards';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { BookEmbedderService } from '../embedding/book-embedder.service';
|
||||
import { BookRepository } from '../book/book.repository';
|
||||
import { LibraryService } from '../library/library.service';
|
||||
import { books } from '../../db/schema';
|
||||
import { AnnCandidate, CandidateMetadata, RecommendationRepository, TargetBookData } from './recommendation.repository';
|
||||
|
||||
@Injectable()
|
||||
export class RecommendationService {
|
||||
private readonly logger = new Logger(RecommendationService.name);
|
||||
|
||||
constructor(
|
||||
private readonly recRepo: RecommendationRepository,
|
||||
private readonly bookRepo: BookRepository,
|
||||
private readonly libraryService: LibraryService,
|
||||
private readonly embedder: BookEmbedderService,
|
||||
) {}
|
||||
|
||||
async getRecommendations(bookId: number, user: RequestUser): Promise<BookRecommendation[]> {
|
||||
const libraryId = await this.bookRepo.findLibraryIdByBookId(bookId);
|
||||
if (libraryId === null) throw new NotFoundException(`Book ${bookId} not found`);
|
||||
await this.libraryService.verifyUserAccess(user.id, libraryId, this.isSuperuser(user));
|
||||
|
||||
const target = await this.recRepo.getTargetBookData(bookId);
|
||||
if (!target) return [];
|
||||
|
||||
let embedding = target.embedding;
|
||||
if (!embedding) {
|
||||
embedding = await this.embedder.embedBook(bookId);
|
||||
}
|
||||
if (!embedding) return [];
|
||||
|
||||
const libs = await this.libraryService.findAll(user);
|
||||
const accessibleLibraryIds = libs.map((l) => l.id);
|
||||
|
||||
const candidates = await this.recRepo.findAnnCandidates(embedding, bookId, accessibleLibraryIds);
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
const candidateMetadata = await this.recRepo.getCandidateMetadata(candidates.map((c) => c.bookId));
|
||||
const metaMap = new Map(candidateMetadata.map((m) => [m.bookId, m]));
|
||||
|
||||
const rescored = candidates
|
||||
.map((c) => ({ bookId: c.bookId, score: this.rescore(c, target, metaMap.get(c.bookId) ?? null) }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, 25);
|
||||
|
||||
if (rescored.length === 0) return [];
|
||||
|
||||
const topIds = rescored.map((r) => r.bookId);
|
||||
const { rows, authorRows, fileRows, genreRows, tagRows, progressRows } = await this.bookRepo.findCards({
|
||||
where: inArray(books.id, topIds),
|
||||
orderBy: [],
|
||||
limit: 25,
|
||||
offset: 0,
|
||||
userId: user.id,
|
||||
});
|
||||
|
||||
const cards = assembleBookCards(rows, authorRows, fileRows, genreRows, tagRows, progressRows);
|
||||
const cardMap = new Map(cards.map((c) => [c.id, c]));
|
||||
|
||||
return rescored.map((r) => ({ book: cardMap.get(r.bookId)!, score: r.score })).filter((r) => r.book != null);
|
||||
}
|
||||
|
||||
private rescore(candidate: AnnCandidate, target: TargetBookData, meta: CandidateMetadata | null): number {
|
||||
const cosineSim = Math.max(0, Math.min(1, candidate.cosineSim));
|
||||
|
||||
const authorSim = meta ? this.jaccard(new Set(target.authorNames), new Set(meta.authorNames)) : 0;
|
||||
const genreTagSim = meta ? this.jaccard(new Set(target.genreTagNames), new Set(meta.genreTagNames)) : 0;
|
||||
|
||||
const seriesBonus = target.seriesName && candidate.seriesName && target.seriesName === candidate.seriesName ? 1.0 : 0.0;
|
||||
|
||||
let ratingProximity = 0.5;
|
||||
if (target.rating != null && candidate.rating != null) {
|
||||
ratingProximity = 1 - Math.abs(target.rating - candidate.rating) / 4;
|
||||
}
|
||||
|
||||
return 0.5 * cosineSim + 0.1 * authorSim + 0.25 * genreTagSim + 0.1 * seriesBonus + 0.05 * ratingProximity;
|
||||
}
|
||||
|
||||
private jaccard(a: Set<string>, b: Set<string>): number {
|
||||
if (a.size === 0 && b.size === 0) return 0;
|
||||
let intersection = 0;
|
||||
for (const x of a) if (b.has(x)) intersection++;
|
||||
return intersection / (a.size + b.size - intersection);
|
||||
}
|
||||
|
||||
private isSuperuser(user: RequestUser): boolean {
|
||||
return user.roles.some((r) => r.isSuperuser);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user