Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): fix migration executor bugs and schema corrections

Browse Source
This commit is contained in:
neon
2026-04-05 20:24:08 -06:00
parent 47e5acd9e6
commit decd0f1a2e
24 changed files with 701 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/features/migration/components/MigrationModal.vue
+63 -2
View File
@@ -66,6 +66,7 @@ interface PathMappingDraft {
interface ReportUnresolvedBook {
sourceBookId: string
title: string | null
author: string | null
reason: string | null
}
@@ -282,7 +283,7 @@ const reportData = computed(() => {
const bookCovers = m('book_covers', 'book_covers')
const coversSkippedAll =
!!bookCovers && bookCovers.imported === 0 && bookCovers.skipped > 0 && bookCovers.unresolved === 0 && bookCovers.failed === 0
const unresolvedBooks = extractPlanUnresolvedBooks(runReport.value?.plan ?? plan.value?.plan ?? null)
const unresolvedBooks = runReport.value?.details.unresolvedBooks ?? extractPlanUnresolvedBooks(runReport.value?.plan ?? plan.value?.plan ?? null)
const startedAt = run.value?.startedAt ? new Date(run.value.startedAt) : null
const endedAt = run.value?.endedAt ? new Date(run.value.endedAt) : null
return {
@@ -300,6 +301,8 @@ const reportData = computed(() => {
bookmarks: m('user_state', 'bookmarks'),
annotations: m('user_state', 'annotations'),
collections: m('user_state', 'collections'),
matchedBooks: runReport.value?.details.matchedBooks ?? [],
userPreview: runReport.value?.details.userPreview ?? [],
unresolvedBooks,
coverFailureCount: bookCovers?.failed ?? 0,
}
@@ -844,6 +847,24 @@ function friendlyUnresolvedReason(reason: string | null): string {
return reason ?? 'Could not determine reason'
}
function friendlyMatchStrategy(strategy: string | null): string {
if (strategy === 'isbn') return 'Matched by ISBN'
if (strategy === 'file_hash') return 'Matched by file hash'
if (strategy === 'path_mapping') return 'Matched by mapped file path'
if (strategy === 'title_author') return 'Matched by title and author'
return strategy ?? 'Match strategy unavailable'
}
function describeUserPreviewCounts(counts: {
statuses: number
fileProgress: number
bookmarks: number
annotations: number
shelves: number
}): string {
return `${counts.statuses} statuses, ${counts.fileProgress} progress entries, ${counts.bookmarks} bookmarks, ${counts.annotations} annotations, ${counts.shelves} collection entries`
}
function extractPlanUnresolvedBooks(planPayload: unknown): ReportUnresolvedBook[] {
const rows = asRecord(planPayload).unresolvedBooks
if (!Array.isArray(rows)) return []
@@ -852,7 +873,7 @@ function extractPlanUnresolvedBooks(planPayload: unknown): ReportUnresolvedBook[
const record = asRecord(row)
const sourceBookId = asString(record.sourceBookId)
if (!sourceBookId) return null
return { sourceBookId, title: asString(record.title), reason: asString(record.reason) }
return { sourceBookId, title: asString(record.title), author: null as string | null, reason: asString(record.reason) }
})
.filter((row): row is ReportUnresolvedBook => row != null)
}
@@ -1490,6 +1511,27 @@ function formatSourceCountLabel(key: string) {
>
Migration completed with no unresolved books or failures.
</div>
<div v-if="reportData.matchedBooks.length > 0" class="space-y-2">
<div>
<p class="text-sm font-semibold">Matched books ({{ reportData.matchedBooks.length }})</p>
<p class="text-xs text-muted-foreground mt-0.5">
These dry-run matches were used to decide which library books received imported data.
</p>
</div>
<div class="space-y-1.5 max-h-56 overflow-y-auto">
<div
v-for="book in reportData.matchedBooks"
:key="`${book.sourceBookId}-${book.targetBookId}`"
class="rounded border border-border px-3 py-2 text-xs"
>
<p class="font-medium text-foreground">{{ book.sourceTitle || `Source book ${book.sourceBookId}` }}</p>
<p v-if="book.sourceAuthor" class="text-muted-foreground mt-0.5">{{ book.sourceAuthor }}</p>
<p class="text-muted-foreground mt-0.5">
{{ friendlyMatchStrategy(book.strategy) }} - {{ book.targetTitle || `Library book ${book.targetBookId}` }}
</p>
</div>
</div>
</div>
<div v-if="reportData.unresolvedBooks.length > 0" class="space-y-2">
<div>
<p class="text-sm font-semibold text-amber-600">Unresolved books ({{ reportData.unresolvedBooks.length }})</p>
@@ -1504,10 +1546,29 @@ function formatSourceCountLabel(key: string) {
class="rounded border border-amber-500/20 bg-amber-500/5 px-3 py-2 text-xs"
>
<p class="font-medium text-foreground">{{ book.title || `Source book ${book.sourceBookId}` }}</p>
<p v-if="book.author" class="text-muted-foreground mt-0.5">{{ book.author }}</p>
<p class="text-muted-foreground mt-0.5">{{ friendlyUnresolvedReason(book.reason) }}</p>
</div>
</div>
</div>
<div v-if="reportData.userPreview.length > 0" class="space-y-2">
<div>
<p class="text-sm font-semibold">Mapped users ({{ reportData.userPreview.length }})</p>
<p class="text-xs text-muted-foreground mt-0.5">
Per-user totals come from the dry-run preview cached with the migration plan.
</p>
</div>
<div class="space-y-1.5">
<div
v-for="preview in reportData.userPreview"
:key="`${preview.sourceUserId}-${preview.targetUserId}`"
class="rounded border border-border px-3 py-2 text-xs"
>
<p class="font-medium text-foreground">{{ preview.username }}</p>
<p class="text-muted-foreground mt-0.5">{{ describeUserPreviewCounts(preview.counts) }}</p>
</div>
</div>
</div>
<div
v-if="reportData.coverFailureCount > 0"
class="rounded border border-red-500/20 bg-red-500/5 px-3 py-2 text-xs text-red-700"
client/src/features/migration/lib/migration-api.ts
+43
View File
@@ -37,9 +37,52 @@ export interface MigrationRunProgress {
export interface MigrationRunReport {
run: MigrationRun
totals: {
processed: number
imported: number
skipped: number
unresolved: number
failed: number
}
metrics: MigrationRunMetric[]
plan: Record<string, unknown> | null
summary: MigrationPlanSummary | null
details: {
matchedBooks: Array<{
sourceBookId: string
sourceTitle: string | null
sourceAuthor: string | null
targetBookId: number
targetTitle: string | null
strategy: string
}>
unresolvedBooks: Array<{
sourceBookId: string
title: string | null
author: string | null
reason: string
}>
duplicateBookMatches: Array<{
targetBookId: number
targetTitle: string | null
sourceBookIds: string[]
sourceTitles: Array<string | null>
strategies: string[]
reason: string
}>
userPreview: Array<{
sourceUserId: string
targetUserId: number
username: string
counts: {
statuses: number
fileProgress: number
bookmarks: number
annotations: number
shelves: number
}
}>
}
}
export interface MigrationWorkflowState {
client/src/features/settings/MigrationSettings.vue
+63 -1
View File
@@ -62,6 +62,7 @@ interface PathMappingDraft {
interface ReportUnresolvedBook {
sourceBookId: string
title: string | null
author: string | null
reason: string | null
}
@@ -264,7 +265,7 @@ const reportData = computed(() => {
const bookCovers = m('book_covers', 'book_covers')
const coversSkippedAll =
!!bookCovers && bookCovers.imported === 0 && bookCovers.skipped > 0 && bookCovers.unresolved === 0 && bookCovers.failed === 0
const unresolvedBooks = extractPlanUnresolvedBooks(runReport.value?.plan ?? plan.value?.plan ?? null)
const unresolvedBooks = runReport.value?.details.unresolvedBooks ?? extractPlanUnresolvedBooks(runReport.value?.plan ?? plan.value?.plan ?? null)
const startedAt = run.value?.startedAt ? new Date(run.value.startedAt) : null
const endedAt = run.value?.endedAt ? new Date(run.value.endedAt) : null
return {
@@ -282,6 +283,8 @@ const reportData = computed(() => {
bookmarks: m('user_state', 'bookmarks'),
annotations: m('user_state', 'annotations'),
collections: m('user_state', 'collections'),
matchedBooks: runReport.value?.details.matchedBooks ?? [],
userPreview: runReport.value?.details.userPreview ?? [],
unresolvedBooks,
coverFailureCount: bookCovers?.failed ?? 0,
}
@@ -854,6 +857,24 @@ function friendlyUnresolvedReason(reason: string | null): string {
return reason ?? 'Could not determine reason'
}
function friendlyMatchStrategy(strategy: string | null): string {
if (strategy === 'isbn') return 'Matched by ISBN'
if (strategy === 'file_hash') return 'Matched by file hash'
if (strategy === 'path_mapping') return 'Matched by mapped file path'
if (strategy === 'title_author') return 'Matched by title and author'
return strategy ?? 'Match strategy unavailable'
}
function describeUserPreviewCounts(counts: {
statuses: number
fileProgress: number
bookmarks: number
annotations: number
shelves: number
}): string {
return `${counts.statuses} statuses, ${counts.fileProgress} progress entries, ${counts.bookmarks} bookmarks, ${counts.annotations} annotations, ${counts.shelves} collection entries`
}
function extractPlanUnresolvedBooks(planPayload: unknown): ReportUnresolvedBook[] {
const rows = asRecord(planPayload).unresolvedBooks
if (!Array.isArray(rows)) return []
@@ -866,6 +887,7 @@ function extractPlanUnresolvedBooks(planPayload: unknown): ReportUnresolvedBook[
return {
sourceBookId,
title: asString(record.title),
author: null as string | null,
reason: asString(record.reason),
}
})
@@ -1480,6 +1502,28 @@ function formatSourceCountLabel(key: string) {
Migration completed with no unresolved books or failures.
</div>
<div v-if="reportData.matchedBooks.length > 0" class="space-y-2">
<div>
<p class="text-sm font-semibold">Matched books ({{ reportData.matchedBooks.length }})</p>
<p class="text-xs text-muted-foreground mt-0.5">
These dry-run matches were used to decide which library books received imported data.
</p>
</div>
<div class="space-y-1.5 max-h-56 overflow-y-auto">
<div
v-for="book in reportData.matchedBooks"
:key="`${book.sourceBookId}-${book.targetBookId}`"
class="rounded border border-border px-3 py-2 text-xs"
>
<p class="font-medium text-foreground">{{ book.sourceTitle || `Source book ${book.sourceBookId}` }}</p>
<p v-if="book.sourceAuthor" class="text-muted-foreground mt-0.5">{{ book.sourceAuthor }}</p>
<p class="text-muted-foreground mt-0.5">
{{ friendlyMatchStrategy(book.strategy) }} - {{ book.targetTitle || `Library book ${book.targetBookId}` }}
</p>
</div>
</div>
</div>
<div v-if="reportData.unresolvedBooks.length > 0" class="space-y-2">
<div>
<p class="text-sm font-semibold text-amber-600">Unresolved books ({{ reportData.unresolvedBooks.length }})</p>
@@ -1494,11 +1538,29 @@ function formatSourceCountLabel(key: string) {
class="rounded border border-amber-500/20 bg-amber-500/5 px-3 py-2 text-xs"
>
<p class="font-medium text-foreground">{{ book.title || `Source book ${book.sourceBookId}` }}</p>
<p v-if="book.author" class="text-muted-foreground mt-0.5">{{ book.author }}</p>
<p class="text-muted-foreground mt-0.5">{{ friendlyUnresolvedReason(book.reason) }}</p>
</div>
</div>
</div>
<div v-if="reportData.userPreview.length > 0" class="space-y-2">
<div>
<p class="text-sm font-semibold">Mapped users ({{ reportData.userPreview.length }})</p>
<p class="text-xs text-muted-foreground mt-0.5">Per-user totals come from the dry-run preview cached with the migration plan.</p>
</div>
<div class="space-y-1.5">
<div
v-for="preview in reportData.userPreview"
:key="`${preview.sourceUserId}-${preview.targetUserId}`"
class="rounded border border-border px-3 py-2 text-xs"
>
<p class="font-medium text-foreground">{{ preview.username }}</p>
<p class="text-muted-foreground mt-0.5">{{ describeUserPreviewCounts(preview.counts) }}</p>
</div>
</div>
</div>
<div v-if="reportData.coverFailureCount > 0" class="rounded border border-red-500/20 bg-red-500/5 px-3 py-2 text-xs text-red-700">
{{ reportData.coverFailureCount }} cover import(s) failed. Detailed failure rows are not stored.
</div>
server/src/db/migrations/0000_initial.sql
+14 -12
View File
@@ -1,5 +1,7 @@
CREATE EXTENSION IF NOT EXISTS vector;--> statement-breakpoint
CREATE EXTENSION IF NOT EXISTS pg_trgm;--> statement-breakpoint
CREATE EXTENSION IF NOT EXISTS vector;
--> statement-breakpoint
CREATE EXTENSION IF NOT EXISTS pg_trgm;
--> statement-breakpoint
CREATE TYPE "public"."library_access_level" AS ENUM('viewer', 'editor', 'owner');--> statement-breakpoint
CREATE TYPE "public"."user_avatar_source" AS ENUM('none', 'external', 'uploaded');--> statement-breakpoint
CREATE TYPE "public"."opds_sort_order" AS ENUM('recent', 'title_asc', 'title_desc', 'author_asc', 'author_desc', 'series_asc', 'series_desc');--> statement-breakpoint
@@ -116,7 +118,8 @@ CREATE TABLE "migration_plan_artifacts" (
"summary" jsonb NOT NULL,
"created_by_user_id" integer,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "migration_plan_artifacts_id_source_profile_unique" UNIQUE("id","source_id","profile_id")
);
--> statement-breakpoint
CREATE TABLE "migration_profiles" (
@@ -130,6 +133,7 @@ CREATE TABLE "migration_profiles" (
"created_by_user_id" integer,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "migration_profiles_id_source_id_unique" UNIQUE("id","source_id"),
CONSTRAINT "migration_profiles_version_positive_chk" CHECK ("migration_profiles"."version" >= 1)
);
--> statement-breakpoint
@@ -218,7 +222,8 @@ CREATE TABLE "library_folders" (
"id" serial PRIMARY KEY NOT NULL,
"library_id" integer NOT NULL,
"path" varchar(4096) NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "library_folders_id_library_id_unique" UNIQUE("id","library_id")
);
--> statement-breakpoint
CREATE TABLE "book_files" (
@@ -251,6 +256,7 @@ CREATE TABLE "books" (
"status" varchar(20) DEFAULT 'present' NOT NULL,
"added_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "books_id_library_folder_id_unique" UNIQUE("id","library_folder_id"),
CONSTRAINT "books_status_chk" CHECK ("books"."status" in ('present', 'missing'))
);
--> statement-breakpoint
@@ -709,6 +715,7 @@ CREATE TABLE "email_providers" (
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "email_providers_user_id_name_unique" UNIQUE("user_id","name"),
CONSTRAINT "email_providers_id_user_id_unique" UNIQUE("id","user_id"),
CONSTRAINT "email_providers_port_range_chk" CHECK ("email_providers"."port" >= 1 and "email_providers"."port" <= 65535)
);
--> statement-breakpoint
@@ -726,7 +733,8 @@ CREATE TABLE "email_recipient_groups" (
"default_template_id" integer,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "email_recipient_groups_user_id_name_unique" UNIQUE("user_id","name")
CONSTRAINT "email_recipient_groups_user_id_name_unique" UNIQUE("user_id","name"),
CONSTRAINT "email_recipient_groups_id_user_id_unique" UNIQUE("id","user_id")
);
--> statement-breakpoint
CREATE TABLE "email_recipients" (
@@ -741,6 +749,7 @@ CREATE TABLE "email_recipients" (
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "email_recipients_user_id_email_unique" UNIQUE("user_id","email"),
CONSTRAINT "email_recipients_id_user_id_unique" UNIQUE("id","user_id"),
CONSTRAINT "email_recipients_device_type_chk" CHECK ("email_recipients"."device_type" is null or "email_recipients"."device_type" in ('kindle', 'kobo', 'other')),
CONSTRAINT "email_recipients_preferred_format_chk" CHECK ("email_recipients"."preferred_format" is null or "email_recipients"."preferred_format" in ('epub', 'pdf', 'mobi', 'azw3', 'cbz', 'cbr'))
);
@@ -889,11 +898,9 @@ CREATE UNIQUE INDEX "users_username_lower_uidx" ON "users" USING btree (lower("u
CREATE UNIQUE INDEX "users_email_lower_uidx" ON "users" USING btree (lower("email")) WHERE "users"."email" is not null;--> statement-breakpoint
CREATE UNIQUE INDEX "users_oidc_subject_issuer_uidx" ON "users" USING btree ("oidc_subject","oidc_issuer") WHERE "users"."oidc_subject" is not null and "users"."oidc_issuer" is not null;--> statement-breakpoint
CREATE UNIQUE INDEX "migration_plan_artifacts_plan_hash_uidx" ON "migration_plan_artifacts" USING btree ("plan_hash");--> statement-breakpoint
CREATE UNIQUE INDEX "migration_plan_artifacts_id_source_profile_uidx" ON "migration_plan_artifacts" USING btree ("id","source_id","profile_id");--> statement-breakpoint
CREATE INDEX "migration_plan_artifacts_source_id_idx" ON "migration_plan_artifacts" USING btree ("source_id");--> statement-breakpoint
CREATE INDEX "migration_plan_artifacts_profile_id_idx" ON "migration_plan_artifacts" USING btree ("profile_id");--> statement-breakpoint
CREATE UNIQUE INDEX "migration_profiles_source_name_version_uidx" ON "migration_profiles" USING btree ("source_id","name","version");--> statement-breakpoint
CREATE UNIQUE INDEX "migration_profiles_id_source_id_uidx" ON "migration_profiles" USING btree ("id","source_id");--> statement-breakpoint
CREATE INDEX "migration_profiles_source_id_idx" ON "migration_profiles" USING btree ("source_id");--> statement-breakpoint
CREATE UNIQUE INDEX "migration_run_metrics_run_stage_entity_uidx" ON "migration_run_metrics" USING btree ("run_id","stage","entity_type");--> statement-breakpoint
CREATE INDEX "migration_run_metrics_run_id_idx" ON "migration_run_metrics" USING btree ("run_id");--> statement-breakpoint
@@ -904,7 +911,6 @@ CREATE INDEX "migration_sources_created_by_user_id_idx" ON "migration_sources" U
CREATE UNIQUE INDEX "libraries_name_lower_uidx" ON "libraries" USING btree (lower("name"));--> statement-breakpoint
CREATE INDEX "library_folders_library_id_idx" ON "library_folders" USING btree ("library_id");--> statement-breakpoint
CREATE UNIQUE INDEX "library_folders_library_path_uidx" ON "library_folders" USING btree ("library_id","path");--> statement-breakpoint
CREATE UNIQUE INDEX "library_folders_id_library_id_uidx" ON "library_folders" USING btree ("id","library_id");--> statement-breakpoint
CREATE UNIQUE INDEX "book_files_absolute_path_uidx" ON "book_files" USING btree ("absolute_path");--> statement-breakpoint
CREATE INDEX "book_files_book_id_idx" ON "book_files" USING btree ("book_id");--> statement-breakpoint
CREATE INDEX "book_files_library_folder_id_idx" ON "book_files" USING btree ("library_folder_id");--> statement-breakpoint
@@ -914,7 +920,6 @@ CREATE INDEX "book_files_format_idx" ON "book_files" USING btree ("format");-->
CREATE INDEX "book_files_library_folder_hash_idx" ON "book_files" USING btree ("library_folder_id","hash");--> statement-breakpoint
CREATE INDEX "book_files_library_folder_ino_idx" ON "book_files" USING btree ("library_folder_id","ino");--> statement-breakpoint
CREATE UNIQUE INDEX "books_library_id_folder_path_idx" ON "books" USING btree ("library_id","folder_path");--> statement-breakpoint
CREATE UNIQUE INDEX "books_id_library_folder_id_uidx" ON "books" USING btree ("id","library_folder_id");--> statement-breakpoint
CREATE INDEX "books_primary_file_id_idx" ON "books" USING btree ("primary_file_id");--> statement-breakpoint
CREATE INDEX "books_library_status_idx" ON "books" USING btree ("library_id","status");--> statement-breakpoint
CREATE INDEX "books_library_added_at_idx" ON "books" USING btree ("library_id","added_at" desc);--> statement-breakpoint
@@ -963,13 +968,10 @@ CREATE INDEX "book_bucket_files_status_idx" ON "book_bucket_files" USING btree (
CREATE INDEX "fwl_book_id_written_at_idx" ON "file_write_log" USING btree ("book_id","written_at");--> statement-breakpoint
CREATE UNIQUE INDEX "email_templates_one_default_per_user_uidx" ON "email_templates" USING btree ("user_id") WHERE "email_templates"."is_default" = true and "email_templates"."user_id" is not null;--> statement-breakpoint
CREATE UNIQUE INDEX "email_templates_system_name_unique" ON "email_templates" USING btree ("name") WHERE "email_templates"."user_id" is null;--> statement-breakpoint
CREATE UNIQUE INDEX "email_providers_id_user_id_uidx" ON "email_providers" USING btree ("id","user_id");--> statement-breakpoint
CREATE UNIQUE INDEX "email_providers_one_default_per_user_uidx" ON "email_providers" USING btree ("user_id") WHERE "email_providers"."is_default" = true and "email_providers"."user_id" is not null;--> statement-breakpoint
CREATE UNIQUE INDEX "email_providers_one_system_provider_uidx" ON "email_providers" USING btree ("is_system_provider") WHERE "email_providers"."is_system_provider" = true;--> statement-breakpoint
CREATE UNIQUE INDEX "email_recipient_groups_id_user_id_uidx" ON "email_recipient_groups" USING btree ("id","user_id");--> statement-breakpoint
CREATE UNIQUE INDEX "email_recipients_user_email_lower_uidx" ON "email_recipients" USING btree ("user_id",lower("email"));--> statement-breakpoint
CREATE UNIQUE INDEX "email_recipients_one_default_per_user_uidx" ON "email_recipients" USING btree ("user_id") WHERE "email_recipients"."is_default" = true;--> statement-breakpoint
CREATE UNIQUE INDEX "email_recipients_id_user_id_uidx" ON "email_recipients" USING btree ("id","user_id");--> statement-breakpoint
CREATE INDEX "email_send_log_user_created_at_idx" ON "email_send_log" USING btree ("user_id","created_at");--> statement-breakpoint
CREATE INDEX "email_send_log_created_at_idx" ON "email_send_log" USING btree ("created_at");--> statement-breakpoint
CREATE INDEX "email_send_log_status_next_retry_idx" ON "email_send_log" USING btree ("status","next_retry_at");
server/src/db/migrations/meta/0000_snapshot.json
+45 -160
View File
@@ -1,5 +1,5 @@
{
"id": "518e0d80-949a-48cf-a150-e83b21bfa2c1",
"id": "67c2c149-d6e1-477c-a7a0-67f819b511ab",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
@@ -941,33 +941,6 @@
"method": "btree",
"with": {}
},
"migration_plan_artifacts_id_source_profile_uidx": {
"name": "migration_plan_artifacts_id_source_profile_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "source_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "profile_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"migration_plan_artifacts_source_id_idx": {
"name": "migration_plan_artifacts_source_id_idx",
"columns": [
@@ -1038,7 +1011,13 @@
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"uniqueConstraints": {
"migration_plan_artifacts_id_source_profile_unique": {
"name": "migration_plan_artifacts_id_source_profile_unique",
"nullsNotDistinct": false,
"columns": ["id", "source_id", "profile_id"]
}
},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
@@ -1141,27 +1120,6 @@
"method": "btree",
"with": {}
},
"migration_profiles_id_source_id_uidx": {
"name": "migration_profiles_id_source_id_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "source_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"migration_profiles_source_id_idx": {
"name": "migration_profiles_source_id_idx",
"columns": [
@@ -1199,7 +1157,13 @@
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"uniqueConstraints": {
"migration_profiles_id_source_id_unique": {
"name": "migration_profiles_id_source_id_unique",
"nullsNotDistinct": false,
"columns": ["id", "source_id"]
}
},
"policies": {},
"checkConstraints": {
"migration_profiles_version_positive_chk": {
@@ -1959,27 +1923,6 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"library_folders_id_library_id_uidx": {
"name": "library_folders_id_library_id_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "library_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -1994,7 +1937,13 @@
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"uniqueConstraints": {
"library_folders_id_library_id_unique": {
"name": "library_folders_id_library_id_unique",
"nullsNotDistinct": false,
"columns": ["id", "library_id"]
}
},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
@@ -2357,27 +2306,6 @@
"method": "btree",
"with": {}
},
"books_id_library_folder_id_uidx": {
"name": "books_id_library_folder_id_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "library_folder_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"books_primary_file_id_idx": {
"name": "books_primary_file_id_idx",
"columns": [
@@ -2475,7 +2403,13 @@
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"uniqueConstraints": {
"books_id_library_folder_id_unique": {
"name": "books_id_library_folder_id_unique",
"nullsNotDistinct": false,
"columns": ["id", "library_folder_id"]
}
},
"policies": {},
"checkConstraints": {
"books_status_chk": {
@@ -6365,27 +6299,6 @@
}
},
"indexes": {
"email_providers_id_user_id_uidx": {
"name": "email_providers_id_user_id_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"email_providers_one_default_per_user_uidx": {
"name": "email_providers_one_default_per_user_uidx",
"columns": [
@@ -6436,6 +6349,11 @@
"name": "email_providers_user_id_name_unique",
"nullsNotDistinct": false,
"columns": ["user_id", "name"]
},
"email_providers_id_user_id_unique": {
"name": "email_providers_id_user_id_unique",
"nullsNotDistinct": false,
"columns": ["id", "user_id"]
}
},
"policies": {},
@@ -6572,29 +6490,7 @@
"default": "now()"
}
},
"indexes": {
"email_recipient_groups_id_user_id_uidx": {
"name": "email_recipient_groups_id_user_id_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"indexes": {},
"foreignKeys": {
"email_recipient_groups_user_id_users_id_fk": {
"name": "email_recipient_groups_user_id_users_id_fk",
@@ -6621,6 +6517,11 @@
"name": "email_recipient_groups_user_id_name_unique",
"nullsNotDistinct": false,
"columns": ["user_id", "name"]
},
"email_recipient_groups_id_user_id_unique": {
"name": "email_recipient_groups_id_user_id_unique",
"nullsNotDistinct": false,
"columns": ["id", "user_id"]
}
},
"policies": {},
@@ -6732,27 +6633,6 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"email_recipients_id_user_id_uidx": {
"name": "email_recipients_id_user_id_uidx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -6781,6 +6661,11 @@
"name": "email_recipients_user_id_email_unique",
"nullsNotDistinct": false,
"columns": ["user_id", "email"]
},
"email_recipients_id_user_id_unique": {
"name": "email_recipients_id_user_id_unique",
"nullsNotDistinct": false,
"columns": ["id", "user_id"]
}
},
"policies": {},
server/src/db/migrations/meta/_journal.json
+1 -1
View File
@@ -5,7 +5,7 @@
{
"idx": 0,
"version": "7",
"when": 1775479477769,
"when": 1775480846078,
"tag": "0000_initial",
"breakpoints": true
}
server/src/db/schema/books.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import { sql } from 'drizzle-orm';
import { bigint, check, foreignKey, index, integer, pgTable, serial, timestamp, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
import { bigint, check, foreignKey, index, integer, pgTable, serial, timestamp, unique, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
import { libraryFolders, libraries } from './libraries';
@@ -24,7 +24,7 @@ export const books = pgTable(
},
(t) => [
uniqueIndex('books_library_id_folder_path_idx').on(t.libraryId, t.folderPath),
uniqueIndex('books_id_library_folder_id_uidx').on(t.id, t.libraryFolderId),
unique('books_id_library_folder_id_unique').on(t.id, t.libraryFolderId),
index('books_primary_file_id_idx').on(t.primaryFileId),
index('books_library_status_idx').on(t.libraryId, t.status),
index('books_library_added_at_idx').on(t.libraryId, sql`${t.addedAt} desc`),
server/src/db/schema/email-providers.ts
+1 -1
View File
@@ -29,7 +29,7 @@ export const emailProviders = pgTable(
},
(t) => [
unique().on(t.userId, t.name),
uniqueIndex('email_providers_id_user_id_uidx').on(t.id, t.userId),
unique('email_providers_id_user_id_unique').on(t.id, t.userId),
uniqueIndex('email_providers_one_default_per_user_uidx')
.on(t.userId)
.where(sql`${t.isDefault} = true and ${t.userId} is not null`),
server/src/db/schema/email-recipients.ts
+2 -2
View File
@@ -29,7 +29,7 @@ export const emailRecipients = pgTable(
uniqueIndex('email_recipients_one_default_per_user_uidx')
.on(t.userId)
.where(sql`${t.isDefault} = true`),
uniqueIndex('email_recipients_id_user_id_uidx').on(t.id, t.userId),
unique('email_recipients_id_user_id_unique').on(t.id, t.userId),
check('email_recipients_device_type_chk', sql`${t.deviceType} is null or ${t.deviceType} in ('kindle', 'kobo', 'other')`),
check(
'email_recipients_preferred_format_chk',
@@ -53,7 +53,7 @@ export const emailRecipientGroups = pgTable(
.notNull()
.$onUpdateFn(() => new Date()),
},
(t) => [unique().on(t.userId, t.name), uniqueIndex('email_recipient_groups_id_user_id_uidx').on(t.id, t.userId)],
(t) => [unique().on(t.userId, t.name), unique('email_recipient_groups_id_user_id_unique').on(t.id, t.userId)],
);
export const emailRecipientGroupMembers = pgTable(
server/src/db/schema/libraries.ts
+16 -2
View File
@@ -1,5 +1,19 @@
import { sql } from 'drizzle-orm';
import { boolean, check, doublePrecision, index, integer, jsonb, pgTable, serial, text, timestamp, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
import {
boolean,
check,
doublePrecision,
index,
integer,
jsonb,
pgTable,
serial,
text,
timestamp,
unique,
uniqueIndex,
varchar,
} from 'drizzle-orm/pg-core';
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride } from '@projectx/types';
export const libraries = pgTable(
@@ -79,7 +93,7 @@ export const libraryFolders = pgTable(
(t) => [
index('library_folders_library_id_idx').on(t.libraryId),
uniqueIndex('library_folders_library_path_uidx').on(t.libraryId, t.path),
uniqueIndex('library_folders_id_library_id_uidx').on(t.id, t.libraryId),
unique('library_folders_id_library_id_unique').on(t.id, t.libraryId),
],
);
server/src/db/schema/migration.ts
+3 -3
View File
@@ -1,4 +1,4 @@
import { check, foreignKey, index, integer, jsonb, pgTable, serial, text, timestamp, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
import { check, foreignKey, index, integer, jsonb, pgTable, serial, text, timestamp, unique, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
import { sql } from 'drizzle-orm';
import { users } from './auth';
@@ -50,7 +50,7 @@ export const migrationProfiles = pgTable(
},
(t) => [
uniqueIndex('migration_profiles_source_name_version_uidx').on(t.sourceId, t.name, t.version),
uniqueIndex('migration_profiles_id_source_id_uidx').on(t.id, t.sourceId),
unique('migration_profiles_id_source_id_unique').on(t.id, t.sourceId),
index('migration_profiles_source_id_idx').on(t.sourceId),
check('migration_profiles_version_positive_chk', sql`${t.version} >= 1`),
],
@@ -81,7 +81,7 @@ export const migrationPlanArtifacts = pgTable(
},
(t) => [
uniqueIndex('migration_plan_artifacts_plan_hash_uidx').on(t.planHash),
uniqueIndex('migration_plan_artifacts_id_source_profile_uidx').on(t.id, t.sourceId, t.profileId),
unique('migration_plan_artifacts_id_source_profile_unique').on(t.id, t.sourceId, t.profileId),
index('migration_plan_artifacts_source_id_idx').on(t.sourceId),
index('migration_plan_artifacts_profile_id_idx').on(t.profileId),
foreignKey({
server/src/modules/migration/executor/cover.importer.ts
+14 -1
View File
@@ -3,9 +3,11 @@ import { constants as fsConstants } from 'fs';
import { access, mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises';
import { join } from 'path';
import type { CoverRefreshedEvent } from '@projectx/types';
import { coverDirPath, generateThumbnail, imageExt } from '../../metadata/lib/cover';
import { MigrationRepository } from '../migration.repository';
import { MigrationImportRepository } from './migration-import.repository';
import { ScanGateway } from '../../scanner/scan.gateway';
import type { PlannerResult } from '../planner/planner.types';
import { type RunStateCheck, emptyCounters, hasErrorCode } from './executor-utils';
@@ -18,6 +20,7 @@ export class CoverImporter {
constructor(
private readonly repo: MigrationRepository,
private readonly importRepo: MigrationImportRepository,
private readonly scanGateway: ScanGateway,
) {}
async import(
@@ -37,16 +40,26 @@ export class CoverImporter {
return;
}
const libraryIdByBookId = await this.importRepo.fetchLibraryIdsByBookIds(matches.map((m) => m.targetBookId));
for (let i = 0; i < matches.length; i += COVER_CONCURRENCY) {
await ensureRunning();
const batch = matches.slice(i, i + COVER_CONCURRENCY);
const results = await Promise.allSettled(batch.map((match) => this.processSingleMatch(runId, match, booksPath, sourceMediaRootPath)));
for (const result of results) {
for (let j = 0; j < results.length; j++) {
const result = results[j];
const match = batch[j];
counters.processed += 1;
if (result.status === 'fulfilled') {
counters[result.value] += 1;
if (result.value === 'imported') {
const libraryId = libraryIdByBookId.get(match.targetBookId);
if (libraryId) {
this.scanGateway.emitCoverRefreshed({ bookId: match.targetBookId, libraryId } satisfies CoverRefreshedEvent);
}
}
} else {
counters.failed += 1;
}
server/src/modules/migration/executor/executor-utils.ts
+18 -4
View File
@@ -99,23 +99,37 @@ export function buildMetadataPatch(sourceBook: {
isbn13: field('isbn13', truncateNullableText(sourceBook.isbn13, 13)),
description: field('description', sourceBook.description),
publisher: field('publisher', truncateNullableText(sourceBook.publisher, 500)),
publishedYear: field('publishedYear', sourceBook.publishedYear),
publishedYear: field('publishedYear', sanitizeBoundedInteger(sourceBook.publishedYear, 1000, 2200)),
language: field('language', truncateNullableText(sourceBook.language, 100)),
pageCount: field('pageCount', sourceBook.pageCount),
pageCount: field('pageCount', sanitizeNonNegativeInteger(sourceBook.pageCount)),
seriesName: field('seriesName', truncateNullableText(sourceBook.seriesName, 500)),
seriesIndex: field('seriesIndex', sourceBook.seriesIndex),
rating: field('rating', sourceBook.rating),
rating: field('rating', sanitizeBoundedInteger(sourceBook.rating, 1, 10)),
googleBooksId: field('googleBooksId', truncateNullableText(sourceBook.googleBooksId, 50)),
goodreadsId: field('goodreadsId', truncateNullableText(sourceBook.goodreadsId, 50)),
amazonId: field('amazonId', truncateNullableText(sourceBook.amazonId, 20)),
hardcoverId: field('hardcoverId', truncateNullableText(sourceBook.hardcoverId, 50)),
audibleId: field('audibleId', truncateNullableText(sourceBook.audibleId, 20)),
comicvineId: field('comicvineId', truncateNullableText(sourceBook.comicvineId, 50)),
durationSeconds: field('durationSeconds', sourceBook.durationSeconds),
durationSeconds: field('durationSeconds', sanitizeNonNegativeInteger(sourceBook.durationSeconds)),
abridged: field('abridged', sourceBook.abridged === undefined ? undefined : (sourceBook.abridged ?? false)),
});
}
function sanitizeBoundedInteger(value: number | null | undefined, min: number, max: number): number | null | undefined {
if (value === undefined) return undefined;
if (value == null) return null;
if (!Number.isFinite(value) || value < min || value > max) return undefined;
return Math.round(value);
}
function sanitizeNonNegativeInteger(value: number | null | undefined): number | null | undefined {
if (value === undefined) return undefined;
if (value == null) return null;
if (!Number.isFinite(value) || value < 0) return undefined;
return Math.round(value);
}
export function buildContributorValues(contributor: {
name: string;
sortName?: string | null;
server/src/modules/migration/executor/migration-executor.service.test.ts
+18 -3
View File
@@ -27,6 +27,12 @@ function createRepoMock() {
};
}
function createScanGatewayMock() {
return {
emitCoverRefreshed: vi.fn(),
};
}
function createImportRepoMock() {
const repo = {
// Single-row methods (kept for backward compat)
@@ -65,6 +71,7 @@ function createImportRepoMock() {
}),
),
fetchTargetBookFiles: vi.fn(() => Promise.resolve(new Map())),
fetchLibraryIdsByBookIds: vi.fn(() => Promise.resolve(new Map())),
// Batch methods
batchUpsertBookMetadata: vi.fn(() => Promise.resolve()),
batchDeleteBookAuthors: vi.fn(() => Promise.resolve()),
@@ -272,7 +279,9 @@ describe('CoverImporter book cover import', () => {
const repo = createRepoMock();
const importRepo = createImportRepoMock();
const importer = new CoverImporter(repo as never, importRepo as never);
const scanGateway = createScanGatewayMock();
importRepo.fetchLibraryIdsByBookIds.mockResolvedValue(new Map([[901, 1]]));
const importer = new CoverImporter(repo as never, importRepo as never, scanGateway as never);
const planned = {
execution: {
@@ -294,6 +303,7 @@ describe('CoverImporter book cover import', () => {
expect(copiedThumbnailBytes.equals(sourceThumbnailBytes)).toBe(true);
expect(importRepo.markCoverAsCustom).toHaveBeenCalledWith(901);
expect(scanGateway.emitCoverRefreshed).toHaveBeenCalledWith({ bookId: 901, libraryId: 1 });
expect(repo.setRunMetric).toHaveBeenCalledWith(
52,
'book_covers',
@@ -316,7 +326,9 @@ describe('CoverImporter book cover import', () => {
const repo = createRepoMock();
const importRepo = createImportRepoMock();
const importer = new CoverImporter(repo as never, importRepo as never);
const scanGateway = createScanGatewayMock();
importRepo.fetchLibraryIdsByBookIds.mockResolvedValue(new Map([[902, 1]]));
const importer = new CoverImporter(repo as never, importRepo as never, scanGateway as never);
const planned = {
execution: {
@@ -328,6 +340,7 @@ describe('CoverImporter book cover import', () => {
const generatedThumbnail = await readFile(join(booksPath, 'covers', '902', 'thumbnail.jpg'));
expect(generatedThumbnail.length).toBeGreaterThan(0);
expect(scanGateway.emitCoverRefreshed).toHaveBeenCalledWith({ bookId: 902, libraryId: 1 });
expect(repo.setRunMetric).toHaveBeenCalledWith(
53,
'book_covers',
@@ -343,7 +356,8 @@ describe('CoverImporter book cover import', () => {
it('skips book cover stage when source media root is not configured', async () => {
const repo = createRepoMock();
const importRepo = createImportRepoMock();
const importer = new CoverImporter(repo as never, importRepo as never);
const scanGateway = createScanGatewayMock();
const importer = new CoverImporter(repo as never, importRepo as never, scanGateway as never);
const planned = {
execution: {
@@ -354,6 +368,7 @@ describe('CoverImporter book cover import', () => {
await importer.import(54, planned as never, '/tmp/books', null, async () => {});
expect(importRepo.markCoverAsCustom).not.toHaveBeenCalled();
expect(scanGateway.emitCoverRefreshed).not.toHaveBeenCalled();
expect(repo.setRunMetric).toHaveBeenCalledWith(
54,
'book_covers',
server/src/modules/migration/executor/migration-executor.service.ts
+2 -1
View File
@@ -121,9 +121,10 @@ export class MigrationExecutorService {
return;
}
const message = error instanceof Error ? error.message : String(error);
const cause = error instanceof Error && error.cause instanceof Error ? ` cause="${error.cause.message}"` : '';
const errorClass = error instanceof Error ? error.constructor.name : 'UnknownError';
this.logger.warn(
`[migration.execute] [fail] runId=${runId} durationMs=${durationMs} errorClass=${errorClass} error="${message}" - migration execution failed`,
`[migration.execute] [fail] runId=${runId} durationMs=${durationMs} errorClass=${errorClass} error="${message}"${cause} - migration execution failed`,
);
const failedStage = await this.failRun(runId, message);
await this.emitRunProgress(runId, 'failed', failedStage);
server/src/modules/migration/executor/migration-import.repository.ts
+16
View File
@@ -546,4 +546,20 @@ export class MigrationImportRepository {
return result;
}
async fetchLibraryIdsByBookIds(bookIds: number[]): Promise<Map<number, number>> {
const result = new Map<number, number>();
if (bookIds.length === 0) return result;
const rows = await this.db
.select({ id: schema.books.id, libraryId: schema.books.libraryId })
.from(schema.books)
.where(inArray(schema.books.id, bookIds));
for (const row of rows) {
result.set(row.id, row.libraryId);
}
return result;
}
}
server/src/modules/migration/migration.module.ts
+2
View File
@@ -4,6 +4,7 @@ import { JwtModule } from '@nestjs/jwt';
import type { StringValue } from 'ms';
import { AuthModule } from '../auth/auth.module';
import { ScannerModule } from '../scanner/scanner.module';
import { MigrationController } from './migration.controller';
import { MigrationProgressGateway } from './migration-progress.gateway';
import { MigrationRepository } from './migration.repository';
@@ -27,6 +28,7 @@ import { MigrationReportingService } from './reporting/migration-reporting.servi
@Module({
imports: [
AuthModule,
ScannerModule,
JwtModule.registerAsync({
inject: [ConfigService],
useFactory: (config: ConfigService) => ({
server/src/modules/migration/reporting/migration-reporting.service.ts
+270 -7
View File
@@ -1,7 +1,10 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import type { SourceBook, SourceExportData } from '../adapters/source-adapter.types';
import { asRecord } from '../core/coerce';
import { MigrationRepository } from '../migration.repository';
import { sanitizeRunForApi } from '../core/api-sanitizers';
import type { PlannedBookMatch, PlannedDuplicateBookMatch, PlannedUserPreview } from '../planner/planner.types';
@Injectable()
export class MigrationReportingService {
@@ -15,12 +18,45 @@ export class MigrationReportingService {
this.repo.listRunMetrics(runId),
run.planArtifactId ? this.repo.findPlanArtifactById(run.planArtifactId) : Promise.resolve(null),
]);
const totals = summarizeMetrics(metrics);
const sourceData = (artifact?.sourceData as SourceExportData | null) ?? null;
const plan = artifact?.plan ?? null;
const summary = artifact?.summary ?? null;
const sourceBooksById = new Map(sourceData?.books.map((book) => [book.sourceBookId, book]) ?? []);
const matchedBooks = normalizeMatchedBooks(asRecord(plan).matchedBooks);
const unresolvedBooks = normalizeUnresolvedBooks(asRecord(plan).unresolvedBooks, sourceBooksById);
const duplicateBookMatches = normalizeDuplicateBookMatches(asRecord(plan).duplicateBookMatches);
const userPreview = normalizeUserPreview(asRecord(plan).userPreview, asRecord(summary).perUserPreview);
const targetTitlesById = await this.repo.findBookTitlesByIds(
uniqueNumbers([...matchedBooks.map((row) => row.targetBookId), ...duplicateBookMatches.map((row) => row.targetBookId)]),
);
return {
run: sanitizeRunForApi(run),
totals,
metrics,
plan: artifact?.plan ?? null,
summary: artifact?.summary ?? null,
plan,
summary,
details: {
matchedBooks: matchedBooks.map((row) => ({
sourceBookId: row.sourceBookId,
sourceTitle: sourceBooksById.get(row.sourceBookId)?.title ?? null,
sourceAuthor: sourceBooksById.get(row.sourceBookId)?.author ?? null,
targetBookId: row.targetBookId,
targetTitle: targetTitlesById.get(row.targetBookId) ?? null,
strategy: row.strategy,
})),
unresolvedBooks,
duplicateBookMatches: duplicateBookMatches.map((row) => ({
targetBookId: row.targetBookId,
targetTitle: targetTitlesById.get(row.targetBookId) ?? null,
sourceBookIds: row.sourceBookIds,
sourceTitles: row.sourceBookIds.map((sourceBookId) => sourceBooksById.get(sourceBookId)?.title ?? null),
strategies: row.strategies,
reason: row.reason,
})),
userPreview,
},
};
}
@@ -71,11 +107,22 @@ export class MigrationReportingService {
section: 'summary',
stage: null,
entityType: null,
processed: report.metrics.reduce((sum, row) => sum + row.processed, 0),
imported: report.metrics.reduce((sum, row) => sum + row.imported, 0),
skipped: report.metrics.reduce((sum, row) => sum + row.skipped, 0),
unresolved: report.metrics.reduce((sum, row) => sum + row.unresolved, 0),
failed: report.metrics.reduce((sum, row) => sum + row.failed, 0),
processed: report.totals.processed,
imported: report.totals.imported,
skipped: report.totals.skipped,
unresolved: report.totals.unresolved,
failed: report.totals.failed,
sourceBookId: null,
sourceTitle: null,
sourceAuthor: null,
targetBookId: null,
targetTitle: null,
strategy: null,
sourceUserId: null,
targetUserId: null,
username: null,
reason: null,
details: `matchedBooks=${report.details.matchedBooks.length}; unresolvedBooks=${report.details.unresolvedBooks.length}; mappedUsers=${report.details.userPreview.length}`,
code: 'summary',
message: `Run state: ${report.run.state}`,
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
@@ -89,10 +136,117 @@ export class MigrationReportingService {
skipped: row.skipped,
unresolved: row.unresolved,
failed: row.failed,
sourceBookId: null,
sourceTitle: null,
sourceAuthor: null,
targetBookId: null,
targetTitle: null,
strategy: null,
sourceUserId: null,
targetUserId: null,
username: null,
reason: null,
details: null,
code: null,
message: null,
createdAt: row.updatedAt.toISOString(),
})),
...report.details.matchedBooks.map((row) => ({
section: 'matched_books',
stage: 'shared_overlays',
entityType: 'book_match',
processed: null,
imported: null,
skipped: null,
unresolved: null,
failed: null,
sourceBookId: row.sourceBookId,
sourceTitle: row.sourceTitle,
sourceAuthor: row.sourceAuthor,
targetBookId: row.targetBookId,
targetTitle: row.targetTitle,
strategy: row.strategy,
sourceUserId: null,
targetUserId: null,
username: null,
reason: null,
details: null,
code: null,
message: null,
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
})),
...report.details.unresolvedBooks.map((row) => ({
section: 'unresolved_books',
stage: null,
entityType: 'book_match',
processed: null,
imported: null,
skipped: null,
unresolved: null,
failed: null,
sourceBookId: row.sourceBookId,
sourceTitle: row.title,
sourceAuthor: row.author,
targetBookId: null,
targetTitle: null,
strategy: null,
sourceUserId: null,
targetUserId: null,
username: null,
reason: row.reason,
details: null,
code: null,
message: null,
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
})),
...report.details.duplicateBookMatches.map((row) => ({
section: 'duplicate_matches',
stage: null,
entityType: 'book_match',
processed: null,
imported: null,
skipped: null,
unresolved: null,
failed: null,
sourceBookId: row.sourceBookIds.join('|'),
sourceTitle: row.sourceTitles.filter((title): title is string => typeof title === 'string').join(' | '),
sourceAuthor: null,
targetBookId: row.targetBookId,
targetTitle: row.targetTitle,
strategy: row.strategies.join('|'),
sourceUserId: null,
targetUserId: null,
username: null,
reason: row.reason,
details: `sourceBookIds=${row.sourceBookIds.join('|')}`,
code: null,
message: null,
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
})),
...report.details.userPreview.map((row) => ({
section: 'user_preview',
stage: 'user_state',
entityType: 'user_preview',
processed: null,
imported: null,
skipped: null,
unresolved: null,
failed: null,
sourceBookId: null,
sourceTitle: null,
sourceAuthor: null,
targetBookId: null,
targetTitle: null,
strategy: null,
sourceUserId: row.sourceUserId,
targetUserId: row.targetUserId,
username: row.username,
reason: null,
details: `statuses=${row.counts.statuses}; fileProgress=${row.counts.fileProgress}; bookmarks=${row.counts.bookmarks}; annotations=${row.counts.annotations}; shelves=${row.counts.shelves}`,
code: null,
message: null,
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
})),
];
return {
@@ -123,3 +277,112 @@ function csvCell(value: string | number | null | undefined): string {
if (!/[",\n]/.test(text)) return text;
return `"${text.replace(/"/g, '""')}"`;
}
function summarizeMetrics(metrics: Array<{ processed: number; imported: number; skipped: number; unresolved: number; failed: number }>) {
return metrics.reduce(
(acc, row) => ({
processed: acc.processed + row.processed,
imported: acc.imported + row.imported,
skipped: acc.skipped + row.skipped,
unresolved: acc.unresolved + row.unresolved,
failed: acc.failed + row.failed,
}),
{
processed: 0,
imported: 0,
skipped: 0,
unresolved: 0,
failed: 0,
},
);
}
function normalizeMatchedBooks(value: unknown): PlannedBookMatch[] {
if (!Array.isArray(value)) return [];
return value.flatMap((row) => {
const record = asRecord(row);
const sourceBookId = typeof record.sourceBookId === 'string' ? record.sourceBookId : null;
const targetBookId = typeof record.targetBookId === 'number' ? record.targetBookId : null;
const strategy = typeof record.strategy === 'string' ? record.strategy : null;
if (!sourceBookId || targetBookId == null || !strategy) return [];
return [{ sourceBookId, targetBookId, strategy: strategy as PlannedBookMatch['strategy'] }];
});
}
function normalizeUnresolvedBooks(
value: unknown,
sourceBooksById: Map<string, SourceBook>,
): Array<{ sourceBookId: string; title: string | null; author: string | null; reason: string }> {
if (!Array.isArray(value)) return [];
return value.flatMap((row) => {
const record = asRecord(row);
const sourceBookId = typeof record.sourceBookId === 'string' ? record.sourceBookId : null;
const reason = typeof record.reason === 'string' ? record.reason : null;
if (!sourceBookId || !reason) return [];
const sourceBook = sourceBooksById.get(sourceBookId);
return [
{
sourceBookId,
title: (typeof record.title === 'string' ? record.title : null) ?? sourceBook?.title ?? null,
author: sourceBook?.author ?? null,
reason,
},
];
});
}
function normalizeDuplicateBookMatches(value: unknown): PlannedDuplicateBookMatch[] {
if (!Array.isArray(value)) return [];
return value.flatMap((row) => {
const record = asRecord(row);
const targetBookId = typeof record.targetBookId === 'number' ? record.targetBookId : null;
const sourceBookIds = Array.isArray(record.sourceBookIds) ? record.sourceBookIds.filter((id): id is string => typeof id === 'string') : [];
const strategies = Array.isArray(record.strategies)
? record.strategies.filter((strategy): strategy is PlannedBookMatch['strategy'] => typeof strategy === 'string')
: [];
if (targetBookId == null || sourceBookIds.length === 0) return [];
return [
{
targetBookId,
sourceBookIds,
matches: sourceBookIds.map((sourceBookId, index) => ({
sourceBookId,
targetBookId,
strategy: strategies[index] ?? 'title_author',
})),
strategies,
reason: 'duplicate_target_match',
} satisfies PlannedDuplicateBookMatch,
];
});
}
function normalizeUserPreview(planValue: unknown, summaryValue: unknown): PlannedUserPreview[] {
const rows = Array.isArray(planValue) ? planValue : Array.isArray(summaryValue) ? summaryValue : [];
return rows.flatMap((row) => {
const record = asRecord(row);
const counts = asRecord(record.counts);
const sourceUserId = typeof record.sourceUserId === 'string' ? record.sourceUserId : null;
const targetUserId = typeof record.targetUserId === 'number' ? record.targetUserId : null;
const username = typeof record.username === 'string' ? record.username : null;
if (!sourceUserId || targetUserId == null || !username) return [];
return [
{
sourceUserId,
targetUserId,
username,
counts: {
statuses: typeof counts.statuses === 'number' ? counts.statuses : 0,
fileProgress: typeof counts.fileProgress === 'number' ? counts.fileProgress : 0,
bookmarks: typeof counts.bookmarks === 'number' ? counts.bookmarks : 0,
annotations: typeof counts.annotations === 'number' ? counts.annotations : 0,
shelves: typeof counts.shelves === 'number' ? counts.shelves : 0,
},
},
];
});
}
function uniqueNumbers(values: number[]): number[] {
return [...new Set(values.filter((value) => Number.isFinite(value)))];
}
server/src/modules/scanner/scanner.module.ts
+1 -1
View File
@@ -29,6 +29,6 @@ import { ScannerService } from './scanner.service';
],
controllers: [ScannerController],
providers: [ScannerService, ScannerRepository, ScanGateway, ScanJobStore, FileEventProcessorService, FileWatcherService],
exports: [ScannerService, FileWatcherService],
exports: [ScannerService, FileWatcherService, ScanGateway],
})
export class ScannerModule {}
server/src/modules/seed/cleanup.service.test.ts
+20
View File
@@ -7,6 +7,7 @@ function makeDb(options?: {
refreshDeleteResult?: { rowCount?: number };
resetDeleteResult?: { rowCount?: number };
oidcDeleteResult?: { rowCount?: number };
authStateTablesPresent?: boolean;
}) {
const refreshDeleteBuilder = {
where: vi.fn().mockResolvedValue(options?.refreshDeleteResult ?? { rowCount: 2 }),
@@ -19,6 +20,12 @@ function makeDb(options?: {
};
return {
execute: vi.fn().mockResolvedValue({
rows:
(options?.authStateTablesPresent ?? true)
? [{ table_name: 'refresh_tokens' }, { table_name: 'password_reset_tokens' }, { table_name: 'oidc_sessions' }]
: [],
}),
delete: vi.fn().mockReturnValueOnce(refreshDeleteBuilder).mockReturnValueOnce(resetDeleteBuilder).mockReturnValueOnce(oidcDeleteBuilder),
__refreshDeleteBuilder: refreshDeleteBuilder,
__resetDeleteBuilder: resetDeleteBuilder,
@@ -48,6 +55,7 @@ describe('CleanupService', () => {
await service.cleanup();
expect(db.execute).toHaveBeenCalledTimes(1);
expect(db.delete).toHaveBeenNthCalledWith(1, schema.refreshTokens);
expect(db.delete).toHaveBeenNthCalledWith(2, schema.passwordResetTokens);
expect(db.delete).toHaveBeenNthCalledWith(3, schema.oidcSessions);
@@ -58,6 +66,18 @@ describe('CleanupService', () => {
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[seed.cleanup_auth_state] [end] refreshDeleted=2 resetDeleted=1 oidcDeleted=3'));
});
it('skips cleanup when auth state tables are missing', async () => {
const db = makeDb({ authStateTablesPresent: false });
const logSpy = vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
const service = new CleanupService(db as never);
await service.cleanup();
expect(db.execute).toHaveBeenCalledTimes(1);
expect(db.delete).not.toHaveBeenCalled();
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[seed.cleanup_auth_state] [end] skipped=true reason=auth_tables_missing'));
});
it('logs failure and rethrows on cleanup errors', async () => {
const db = makeDb();
const err = new Error('delete failed');
server/src/modules/seed/cleanup.service.ts
+16
View File
@@ -1,5 +1,6 @@
import { Inject, Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { sql } from 'drizzle-orm';
import { eq, isNotNull, lt, or } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -17,12 +18,27 @@ export class CleanupService implements OnApplicationBootstrap {
await this.cleanup();
}
private async hasAuthStateTables(): Promise<boolean> {
const result = await this.db.execute<{ table_name: string }>(sql`
select table_name
from information_schema.tables
where table_schema = 'public'
and table_name in ('refresh_tokens', 'password_reset_tokens', 'oidc_sessions')
`);
return result.rows.length === 3;
}
@Cron('0 3 * * *')
async cleanup() {
const startedAt = Date.now();
const event = 'seed.cleanup_auth_state';
this.logger.log(`[${event}] [start] - auth state cleanup started`);
try {
if (!(await this.hasAuthStateTables())) {
this.logger.log(`[${event}] [end] skipped=true reason=auth_tables_missing durationMs=${Date.now() - startedAt} - auth state cleanup skipped`);
return;
}
const now = new Date();
const { rowCount: refreshCount } = await this.db
server/test/e2e/migration-booklore/migration-booklore-fixture-builder.ts
+7 -7
View File
@@ -272,19 +272,19 @@ export async function seedCanonicalScenario(ctx: MigrationBookloreE2EContext): P
null,
'Hash description',
null,
101,
null,
-5,
null,
null,
12,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
-30,
0,
null,
],
server/test/e2e/migration-booklore/migration-booklore-harness.ts
+4
View File
@@ -216,9 +216,11 @@ export async function waitForMigrationToFinish(
};
report: {
run: { id: number; state: string; currentStage: string | null };
totals: { processed: number; imported: number; skipped: number; unresolved: number; failed: number };
metrics: Array<Record<string, unknown>>;
plan: Record<string, unknown> | null;
summary: Record<string, unknown> | null;
details: Record<string, unknown>;
};
}> {
let latestProgress:
@@ -255,9 +257,11 @@ export async function waitForMigrationToFinish(
const reportResponse = await apiJson<{
run: { id: number; state: string; currentStage: string | null };
totals: { processed: number; imported: number; skipped: number; unresolved: number; failed: number };
metrics: Array<Record<string, unknown>>;
plan: Record<string, unknown> | null;
summary: Record<string, unknown> | null;
details: Record<string, unknown>;
}>(ctx, {
method: 'GET',
url: `/api/v1/migration/runs/${runId}/report`,
server/test/migration-booklore.e2e-spec.ts
+60 -2
View File
@@ -369,7 +369,25 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
});
expect(jsonExport.statusCode).toBe(200);
expect(jsonExport.headers['content-type']).toContain('application/json');
expect(jsonExport.body).toContain('"state": "completed"');
const parsedJsonExport = JSON.parse(jsonExport.body) as {
run: { state: string };
details: {
matchedBooks: Array<{ sourceBookId: string; sourceTitle: string | null; targetBookId: number; strategy: string }>;
userPreview: unknown[];
};
};
expect(parsedJsonExport.run.state).toBe('completed');
expect(parsedJsonExport.details.matchedBooks).toEqual(
expect.arrayContaining([
expect.objectContaining({
sourceBookId: '101',
sourceTitle: 'Imported ISBN Title',
targetBookId: scenario.books.isbn.bookId,
strategy: 'isbn',
}),
]),
);
expect(parsedJsonExport.details.userPreview).toHaveLength(2);
const csvExport = await apiText(ctx, {
method: 'GET',
@@ -378,7 +396,44 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
});
expect(csvExport.statusCode).toBe(200);
expect(csvExport.headers['content-type']).toContain('text/csv');
expect(csvExport.body).toContain('section,stage,entityType,processed,imported,skipped,unresolved,failed,code,message,createdAt');
expect(csvExport.body).toContain(
'section,stage,entityType,processed,imported,skipped,unresolved,failed,sourceBookId,sourceTitle,sourceAuthor,targetBookId,targetTitle,strategy,sourceUserId,targetUserId,username,reason,details,code,message,createdAt',
);
expect(csvExport.body).toContain('matched_books,shared_overlays,book_match');
expect(csvExport.body).toContain('Imported ISBN Title');
const reportDetails = finished.report.details as {
matchedBooks?: Array<{ sourceBookId: string; sourceTitle: string | null; targetBookId: number; strategy: string }>;
userPreview?: Array<{ username: string; counts: Record<string, number> }>;
};
expect(reportDetails.matchedBooks).toEqual(
expect.arrayContaining([
expect.objectContaining({
sourceBookId: '101',
sourceTitle: 'Imported ISBN Title',
targetBookId: scenario.books.isbn.bookId,
strategy: 'isbn',
}),
expect.objectContaining({
sourceBookId: '102',
sourceTitle: 'Imported Hash Title',
targetBookId: scenario.books.hash.bookId,
strategy: 'file_hash',
}),
]),
);
expect(reportDetails.userPreview).toEqual(
expect.arrayContaining([
expect.objectContaining({
username: 'alice',
counts: expect.objectContaining({ statuses: 2, fileProgress: 2, bookmarks: 1, annotations: 1, shelves: 2 }),
}),
expect.objectContaining({
username: 'bob',
counts: expect.objectContaining({ statuses: 1, fileProgress: 1, bookmarks: 1, annotations: 1, shelves: 1 }),
}),
]),
);
const metricByKey = buildMetricMap(finished.report.metrics as MetricRow[]);
expect(metricByKey.get('shared_overlays:book_metadata')).toMatchObject({
@@ -478,8 +533,11 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
title: 'Imported Hash Title',
description: 'Hash description',
subtitle: null,
publishedYear: null,
pageCount: null,
rating: null,
googleBooksId: null,
durationSeconds: null,
});
expect(metadataByBookId.get(scenario.books.audio.bookId)).toMatchObject({
title: 'Audio Path Match',