mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
fix(server): fix migration executor bugs and schema corrections
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
CREATE EXTENSION IF NOT EXISTS vector;--> statement-breakpoint
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;--> statement-breakpoint
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
--> statement-breakpoint
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
--> statement-breakpoint
|
||||
CREATE TYPE "public"."library_access_level" AS ENUM('viewer', 'editor', 'owner');--> statement-breakpoint
|
||||
CREATE TYPE "public"."user_avatar_source" AS ENUM('none', 'external', 'uploaded');--> statement-breakpoint
|
||||
CREATE TYPE "public"."opds_sort_order" AS ENUM('recent', 'title_asc', 'title_desc', 'author_asc', 'author_desc', 'series_asc', 'series_desc');--> statement-breakpoint
|
||||
@@ -116,7 +118,8 @@ CREATE TABLE "migration_plan_artifacts" (
|
||||
"summary" jsonb NOT NULL,
|
||||
"created_by_user_id" integer,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "migration_plan_artifacts_id_source_profile_unique" UNIQUE("id","source_id","profile_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "migration_profiles" (
|
||||
@@ -130,6 +133,7 @@ CREATE TABLE "migration_profiles" (
|
||||
"created_by_user_id" integer,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "migration_profiles_id_source_id_unique" UNIQUE("id","source_id"),
|
||||
CONSTRAINT "migration_profiles_version_positive_chk" CHECK ("migration_profiles"."version" >= 1)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
@@ -218,7 +222,8 @@ CREATE TABLE "library_folders" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"library_id" integer NOT NULL,
|
||||
"path" varchar(4096) NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "library_folders_id_library_id_unique" UNIQUE("id","library_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "book_files" (
|
||||
@@ -251,6 +256,7 @@ CREATE TABLE "books" (
|
||||
"status" varchar(20) DEFAULT 'present' NOT NULL,
|
||||
"added_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "books_id_library_folder_id_unique" UNIQUE("id","library_folder_id"),
|
||||
CONSTRAINT "books_status_chk" CHECK ("books"."status" in ('present', 'missing'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
@@ -709,6 +715,7 @@ CREATE TABLE "email_providers" (
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "email_providers_user_id_name_unique" UNIQUE("user_id","name"),
|
||||
CONSTRAINT "email_providers_id_user_id_unique" UNIQUE("id","user_id"),
|
||||
CONSTRAINT "email_providers_port_range_chk" CHECK ("email_providers"."port" >= 1 and "email_providers"."port" <= 65535)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
@@ -726,7 +733,8 @@ CREATE TABLE "email_recipient_groups" (
|
||||
"default_template_id" integer,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "email_recipient_groups_user_id_name_unique" UNIQUE("user_id","name")
|
||||
CONSTRAINT "email_recipient_groups_user_id_name_unique" UNIQUE("user_id","name"),
|
||||
CONSTRAINT "email_recipient_groups_id_user_id_unique" UNIQUE("id","user_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "email_recipients" (
|
||||
@@ -741,6 +749,7 @@ CREATE TABLE "email_recipients" (
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "email_recipients_user_id_email_unique" UNIQUE("user_id","email"),
|
||||
CONSTRAINT "email_recipients_id_user_id_unique" UNIQUE("id","user_id"),
|
||||
CONSTRAINT "email_recipients_device_type_chk" CHECK ("email_recipients"."device_type" is null or "email_recipients"."device_type" in ('kindle', 'kobo', 'other')),
|
||||
CONSTRAINT "email_recipients_preferred_format_chk" CHECK ("email_recipients"."preferred_format" is null or "email_recipients"."preferred_format" in ('epub', 'pdf', 'mobi', 'azw3', 'cbz', 'cbr'))
|
||||
);
|
||||
@@ -889,11 +898,9 @@ CREATE UNIQUE INDEX "users_username_lower_uidx" ON "users" USING btree (lower("u
|
||||
CREATE UNIQUE INDEX "users_email_lower_uidx" ON "users" USING btree (lower("email")) WHERE "users"."email" is not null;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "users_oidc_subject_issuer_uidx" ON "users" USING btree ("oidc_subject","oidc_issuer") WHERE "users"."oidc_subject" is not null and "users"."oidc_issuer" is not null;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "migration_plan_artifacts_plan_hash_uidx" ON "migration_plan_artifacts" USING btree ("plan_hash");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "migration_plan_artifacts_id_source_profile_uidx" ON "migration_plan_artifacts" USING btree ("id","source_id","profile_id");--> statement-breakpoint
|
||||
CREATE INDEX "migration_plan_artifacts_source_id_idx" ON "migration_plan_artifacts" USING btree ("source_id");--> statement-breakpoint
|
||||
CREATE INDEX "migration_plan_artifacts_profile_id_idx" ON "migration_plan_artifacts" USING btree ("profile_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "migration_profiles_source_name_version_uidx" ON "migration_profiles" USING btree ("source_id","name","version");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "migration_profiles_id_source_id_uidx" ON "migration_profiles" USING btree ("id","source_id");--> statement-breakpoint
|
||||
CREATE INDEX "migration_profiles_source_id_idx" ON "migration_profiles" USING btree ("source_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "migration_run_metrics_run_stage_entity_uidx" ON "migration_run_metrics" USING btree ("run_id","stage","entity_type");--> statement-breakpoint
|
||||
CREATE INDEX "migration_run_metrics_run_id_idx" ON "migration_run_metrics" USING btree ("run_id");--> statement-breakpoint
|
||||
@@ -904,7 +911,6 @@ CREATE INDEX "migration_sources_created_by_user_id_idx" ON "migration_sources" U
|
||||
CREATE UNIQUE INDEX "libraries_name_lower_uidx" ON "libraries" USING btree (lower("name"));--> statement-breakpoint
|
||||
CREATE INDEX "library_folders_library_id_idx" ON "library_folders" USING btree ("library_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "library_folders_library_path_uidx" ON "library_folders" USING btree ("library_id","path");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "library_folders_id_library_id_uidx" ON "library_folders" USING btree ("id","library_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "book_files_absolute_path_uidx" ON "book_files" USING btree ("absolute_path");--> statement-breakpoint
|
||||
CREATE INDEX "book_files_book_id_idx" ON "book_files" USING btree ("book_id");--> statement-breakpoint
|
||||
CREATE INDEX "book_files_library_folder_id_idx" ON "book_files" USING btree ("library_folder_id");--> statement-breakpoint
|
||||
@@ -914,7 +920,6 @@ CREATE INDEX "book_files_format_idx" ON "book_files" USING btree ("format");-->
|
||||
CREATE INDEX "book_files_library_folder_hash_idx" ON "book_files" USING btree ("library_folder_id","hash");--> statement-breakpoint
|
||||
CREATE INDEX "book_files_library_folder_ino_idx" ON "book_files" USING btree ("library_folder_id","ino");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "books_library_id_folder_path_idx" ON "books" USING btree ("library_id","folder_path");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "books_id_library_folder_id_uidx" ON "books" USING btree ("id","library_folder_id");--> statement-breakpoint
|
||||
CREATE INDEX "books_primary_file_id_idx" ON "books" USING btree ("primary_file_id");--> statement-breakpoint
|
||||
CREATE INDEX "books_library_status_idx" ON "books" USING btree ("library_id","status");--> statement-breakpoint
|
||||
CREATE INDEX "books_library_added_at_idx" ON "books" USING btree ("library_id","added_at" desc);--> statement-breakpoint
|
||||
@@ -963,13 +968,10 @@ CREATE INDEX "book_bucket_files_status_idx" ON "book_bucket_files" USING btree (
|
||||
CREATE INDEX "fwl_book_id_written_at_idx" ON "file_write_log" USING btree ("book_id","written_at");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_templates_one_default_per_user_uidx" ON "email_templates" USING btree ("user_id") WHERE "email_templates"."is_default" = true and "email_templates"."user_id" is not null;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_templates_system_name_unique" ON "email_templates" USING btree ("name") WHERE "email_templates"."user_id" is null;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_providers_id_user_id_uidx" ON "email_providers" USING btree ("id","user_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_providers_one_default_per_user_uidx" ON "email_providers" USING btree ("user_id") WHERE "email_providers"."is_default" = true and "email_providers"."user_id" is not null;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_providers_one_system_provider_uidx" ON "email_providers" USING btree ("is_system_provider") WHERE "email_providers"."is_system_provider" = true;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_recipient_groups_id_user_id_uidx" ON "email_recipient_groups" USING btree ("id","user_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_recipients_user_email_lower_uidx" ON "email_recipients" USING btree ("user_id",lower("email"));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_recipients_one_default_per_user_uidx" ON "email_recipients" USING btree ("user_id") WHERE "email_recipients"."is_default" = true;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "email_recipients_id_user_id_uidx" ON "email_recipients" USING btree ("id","user_id");--> statement-breakpoint
|
||||
CREATE INDEX "email_send_log_user_created_at_idx" ON "email_send_log" USING btree ("user_id","created_at");--> statement-breakpoint
|
||||
CREATE INDEX "email_send_log_created_at_idx" ON "email_send_log" USING btree ("created_at");--> statement-breakpoint
|
||||
CREATE INDEX "email_send_log_status_next_retry_idx" ON "email_send_log" USING btree ("status","next_retry_at");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"id": "518e0d80-949a-48cf-a150-e83b21bfa2c1",
|
||||
"id": "67c2c149-d6e1-477c-a7a0-67f819b511ab",
|
||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
@@ -941,33 +941,6 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"migration_plan_artifacts_id_source_profile_uidx": {
|
||||
"name": "migration_plan_artifacts_id_source_profile_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "source_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "profile_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"migration_plan_artifacts_source_id_idx": {
|
||||
"name": "migration_plan_artifacts_source_id_idx",
|
||||
"columns": [
|
||||
@@ -1038,7 +1011,13 @@
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"uniqueConstraints": {
|
||||
"migration_plan_artifacts_id_source_profile_unique": {
|
||||
"name": "migration_plan_artifacts_id_source_profile_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "source_id", "profile_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
@@ -1141,27 +1120,6 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"migration_profiles_id_source_id_uidx": {
|
||||
"name": "migration_profiles_id_source_id_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "source_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"migration_profiles_source_id_idx": {
|
||||
"name": "migration_profiles_source_id_idx",
|
||||
"columns": [
|
||||
@@ -1199,7 +1157,13 @@
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"uniqueConstraints": {
|
||||
"migration_profiles_id_source_id_unique": {
|
||||
"name": "migration_profiles_id_source_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "source_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
"checkConstraints": {
|
||||
"migration_profiles_version_positive_chk": {
|
||||
@@ -1959,27 +1923,6 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"library_folders_id_library_id_uidx": {
|
||||
"name": "library_folders_id_library_id_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "library_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -1994,7 +1937,13 @@
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"uniqueConstraints": {
|
||||
"library_folders_id_library_id_unique": {
|
||||
"name": "library_folders_id_library_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "library_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
@@ -2357,27 +2306,6 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"books_id_library_folder_id_uidx": {
|
||||
"name": "books_id_library_folder_id_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "library_folder_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"books_primary_file_id_idx": {
|
||||
"name": "books_primary_file_id_idx",
|
||||
"columns": [
|
||||
@@ -2475,7 +2403,13 @@
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"uniqueConstraints": {
|
||||
"books_id_library_folder_id_unique": {
|
||||
"name": "books_id_library_folder_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "library_folder_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
"checkConstraints": {
|
||||
"books_status_chk": {
|
||||
@@ -6365,27 +6299,6 @@
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"email_providers_id_user_id_uidx": {
|
||||
"name": "email_providers_id_user_id_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "user_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"email_providers_one_default_per_user_uidx": {
|
||||
"name": "email_providers_one_default_per_user_uidx",
|
||||
"columns": [
|
||||
@@ -6436,6 +6349,11 @@
|
||||
"name": "email_providers_user_id_name_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["user_id", "name"]
|
||||
},
|
||||
"email_providers_id_user_id_unique": {
|
||||
"name": "email_providers_id_user_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "user_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
@@ -6572,29 +6490,7 @@
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"email_recipient_groups_id_user_id_uidx": {
|
||||
"name": "email_recipient_groups_id_user_id_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "user_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {
|
||||
"email_recipient_groups_user_id_users_id_fk": {
|
||||
"name": "email_recipient_groups_user_id_users_id_fk",
|
||||
@@ -6621,6 +6517,11 @@
|
||||
"name": "email_recipient_groups_user_id_name_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["user_id", "name"]
|
||||
},
|
||||
"email_recipient_groups_id_user_id_unique": {
|
||||
"name": "email_recipient_groups_id_user_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "user_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
@@ -6732,27 +6633,6 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"email_recipients_id_user_id_uidx": {
|
||||
"name": "email_recipients_id_user_id_uidx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "user_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -6781,6 +6661,11 @@
|
||||
"name": "email_recipients_user_id_email_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["user_id", "email"]
|
||||
},
|
||||
"email_recipients_id_user_id_unique": {
|
||||
"name": "email_recipients_id_user_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": ["id", "user_id"]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1775479477769,
|
||||
"when": 1775480846078,
|
||||
"tag": "0000_initial",
|
||||
"breakpoints": true
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { bigint, check, foreignKey, index, integer, pgTable, serial, timestamp, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
|
||||
import { bigint, check, foreignKey, index, integer, pgTable, serial, timestamp, unique, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
|
||||
|
||||
import { libraryFolders, libraries } from './libraries';
|
||||
|
||||
@@ -24,7 +24,7 @@ export const books = pgTable(
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('books_library_id_folder_path_idx').on(t.libraryId, t.folderPath),
|
||||
uniqueIndex('books_id_library_folder_id_uidx').on(t.id, t.libraryFolderId),
|
||||
unique('books_id_library_folder_id_unique').on(t.id, t.libraryFolderId),
|
||||
index('books_primary_file_id_idx').on(t.primaryFileId),
|
||||
index('books_library_status_idx').on(t.libraryId, t.status),
|
||||
index('books_library_added_at_idx').on(t.libraryId, sql`${t.addedAt} desc`),
|
||||
|
||||
@@ -29,7 +29,7 @@ export const emailProviders = pgTable(
|
||||
},
|
||||
(t) => [
|
||||
unique().on(t.userId, t.name),
|
||||
uniqueIndex('email_providers_id_user_id_uidx').on(t.id, t.userId),
|
||||
unique('email_providers_id_user_id_unique').on(t.id, t.userId),
|
||||
uniqueIndex('email_providers_one_default_per_user_uidx')
|
||||
.on(t.userId)
|
||||
.where(sql`${t.isDefault} = true and ${t.userId} is not null`),
|
||||
|
||||
@@ -29,7 +29,7 @@ export const emailRecipients = pgTable(
|
||||
uniqueIndex('email_recipients_one_default_per_user_uidx')
|
||||
.on(t.userId)
|
||||
.where(sql`${t.isDefault} = true`),
|
||||
uniqueIndex('email_recipients_id_user_id_uidx').on(t.id, t.userId),
|
||||
unique('email_recipients_id_user_id_unique').on(t.id, t.userId),
|
||||
check('email_recipients_device_type_chk', sql`${t.deviceType} is null or ${t.deviceType} in ('kindle', 'kobo', 'other')`),
|
||||
check(
|
||||
'email_recipients_preferred_format_chk',
|
||||
@@ -53,7 +53,7 @@ export const emailRecipientGroups = pgTable(
|
||||
.notNull()
|
||||
.$onUpdateFn(() => new Date()),
|
||||
},
|
||||
(t) => [unique().on(t.userId, t.name), uniqueIndex('email_recipient_groups_id_user_id_uidx').on(t.id, t.userId)],
|
||||
(t) => [unique().on(t.userId, t.name), unique('email_recipient_groups_id_user_id_unique').on(t.id, t.userId)],
|
||||
);
|
||||
|
||||
export const emailRecipientGroupMembers = pgTable(
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { boolean, check, doublePrecision, index, integer, jsonb, pgTable, serial, text, timestamp, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
|
||||
import {
|
||||
boolean,
|
||||
check,
|
||||
doublePrecision,
|
||||
index,
|
||||
integer,
|
||||
jsonb,
|
||||
pgTable,
|
||||
serial,
|
||||
text,
|
||||
timestamp,
|
||||
unique,
|
||||
uniqueIndex,
|
||||
varchar,
|
||||
} from 'drizzle-orm/pg-core';
|
||||
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride } from '@projectx/types';
|
||||
|
||||
export const libraries = pgTable(
|
||||
@@ -79,7 +93,7 @@ export const libraryFolders = pgTable(
|
||||
(t) => [
|
||||
index('library_folders_library_id_idx').on(t.libraryId),
|
||||
uniqueIndex('library_folders_library_path_uidx').on(t.libraryId, t.path),
|
||||
uniqueIndex('library_folders_id_library_id_uidx').on(t.id, t.libraryId),
|
||||
unique('library_folders_id_library_id_unique').on(t.id, t.libraryId),
|
||||
],
|
||||
);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { check, foreignKey, index, integer, jsonb, pgTable, serial, text, timestamp, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
|
||||
import { check, foreignKey, index, integer, jsonb, pgTable, serial, text, timestamp, unique, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
|
||||
import { sql } from 'drizzle-orm';
|
||||
|
||||
import { users } from './auth';
|
||||
@@ -50,7 +50,7 @@ export const migrationProfiles = pgTable(
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('migration_profiles_source_name_version_uidx').on(t.sourceId, t.name, t.version),
|
||||
uniqueIndex('migration_profiles_id_source_id_uidx').on(t.id, t.sourceId),
|
||||
unique('migration_profiles_id_source_id_unique').on(t.id, t.sourceId),
|
||||
index('migration_profiles_source_id_idx').on(t.sourceId),
|
||||
check('migration_profiles_version_positive_chk', sql`${t.version} >= 1`),
|
||||
],
|
||||
@@ -81,7 +81,7 @@ export const migrationPlanArtifacts = pgTable(
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('migration_plan_artifacts_plan_hash_uidx').on(t.planHash),
|
||||
uniqueIndex('migration_plan_artifacts_id_source_profile_uidx').on(t.id, t.sourceId, t.profileId),
|
||||
unique('migration_plan_artifacts_id_source_profile_unique').on(t.id, t.sourceId, t.profileId),
|
||||
index('migration_plan_artifacts_source_id_idx').on(t.sourceId),
|
||||
index('migration_plan_artifacts_profile_id_idx').on(t.profileId),
|
||||
foreignKey({
|
||||
|
||||
@@ -3,9 +3,11 @@ import { constants as fsConstants } from 'fs';
|
||||
import { access, mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
import type { CoverRefreshedEvent } from '@projectx/types';
|
||||
import { coverDirPath, generateThumbnail, imageExt } from '../../metadata/lib/cover';
|
||||
import { MigrationRepository } from '../migration.repository';
|
||||
import { MigrationImportRepository } from './migration-import.repository';
|
||||
import { ScanGateway } from '../../scanner/scan.gateway';
|
||||
import type { PlannerResult } from '../planner/planner.types';
|
||||
import { type RunStateCheck, emptyCounters, hasErrorCode } from './executor-utils';
|
||||
|
||||
@@ -18,6 +20,7 @@ export class CoverImporter {
|
||||
constructor(
|
||||
private readonly repo: MigrationRepository,
|
||||
private readonly importRepo: MigrationImportRepository,
|
||||
private readonly scanGateway: ScanGateway,
|
||||
) {}
|
||||
|
||||
async import(
|
||||
@@ -37,16 +40,26 @@ export class CoverImporter {
|
||||
return;
|
||||
}
|
||||
|
||||
const libraryIdByBookId = await this.importRepo.fetchLibraryIdsByBookIds(matches.map((m) => m.targetBookId));
|
||||
|
||||
for (let i = 0; i < matches.length; i += COVER_CONCURRENCY) {
|
||||
await ensureRunning();
|
||||
const batch = matches.slice(i, i + COVER_CONCURRENCY);
|
||||
|
||||
const results = await Promise.allSettled(batch.map((match) => this.processSingleMatch(runId, match, booksPath, sourceMediaRootPath)));
|
||||
|
||||
for (const result of results) {
|
||||
for (let j = 0; j < results.length; j++) {
|
||||
const result = results[j];
|
||||
const match = batch[j];
|
||||
counters.processed += 1;
|
||||
if (result.status === 'fulfilled') {
|
||||
counters[result.value] += 1;
|
||||
if (result.value === 'imported') {
|
||||
const libraryId = libraryIdByBookId.get(match.targetBookId);
|
||||
if (libraryId) {
|
||||
this.scanGateway.emitCoverRefreshed({ bookId: match.targetBookId, libraryId } satisfies CoverRefreshedEvent);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
counters.failed += 1;
|
||||
}
|
||||
|
||||
@@ -99,23 +99,37 @@ export function buildMetadataPatch(sourceBook: {
|
||||
isbn13: field('isbn13', truncateNullableText(sourceBook.isbn13, 13)),
|
||||
description: field('description', sourceBook.description),
|
||||
publisher: field('publisher', truncateNullableText(sourceBook.publisher, 500)),
|
||||
publishedYear: field('publishedYear', sourceBook.publishedYear),
|
||||
publishedYear: field('publishedYear', sanitizeBoundedInteger(sourceBook.publishedYear, 1000, 2200)),
|
||||
language: field('language', truncateNullableText(sourceBook.language, 100)),
|
||||
pageCount: field('pageCount', sourceBook.pageCount),
|
||||
pageCount: field('pageCount', sanitizeNonNegativeInteger(sourceBook.pageCount)),
|
||||
seriesName: field('seriesName', truncateNullableText(sourceBook.seriesName, 500)),
|
||||
seriesIndex: field('seriesIndex', sourceBook.seriesIndex),
|
||||
rating: field('rating', sourceBook.rating),
|
||||
rating: field('rating', sanitizeBoundedInteger(sourceBook.rating, 1, 10)),
|
||||
googleBooksId: field('googleBooksId', truncateNullableText(sourceBook.googleBooksId, 50)),
|
||||
goodreadsId: field('goodreadsId', truncateNullableText(sourceBook.goodreadsId, 50)),
|
||||
amazonId: field('amazonId', truncateNullableText(sourceBook.amazonId, 20)),
|
||||
hardcoverId: field('hardcoverId', truncateNullableText(sourceBook.hardcoverId, 50)),
|
||||
audibleId: field('audibleId', truncateNullableText(sourceBook.audibleId, 20)),
|
||||
comicvineId: field('comicvineId', truncateNullableText(sourceBook.comicvineId, 50)),
|
||||
durationSeconds: field('durationSeconds', sourceBook.durationSeconds),
|
||||
durationSeconds: field('durationSeconds', sanitizeNonNegativeInteger(sourceBook.durationSeconds)),
|
||||
abridged: field('abridged', sourceBook.abridged === undefined ? undefined : (sourceBook.abridged ?? false)),
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizeBoundedInteger(value: number | null | undefined, min: number, max: number): number | null | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (value == null) return null;
|
||||
if (!Number.isFinite(value) || value < min || value > max) return undefined;
|
||||
return Math.round(value);
|
||||
}
|
||||
|
||||
function sanitizeNonNegativeInteger(value: number | null | undefined): number | null | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (value == null) return null;
|
||||
if (!Number.isFinite(value) || value < 0) return undefined;
|
||||
return Math.round(value);
|
||||
}
|
||||
|
||||
export function buildContributorValues(contributor: {
|
||||
name: string;
|
||||
sortName?: string | null;
|
||||
|
||||
@@ -27,6 +27,12 @@ function createRepoMock() {
|
||||
};
|
||||
}
|
||||
|
||||
function createScanGatewayMock() {
|
||||
return {
|
||||
emitCoverRefreshed: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createImportRepoMock() {
|
||||
const repo = {
|
||||
// Single-row methods (kept for backward compat)
|
||||
@@ -65,6 +71,7 @@ function createImportRepoMock() {
|
||||
}),
|
||||
),
|
||||
fetchTargetBookFiles: vi.fn(() => Promise.resolve(new Map())),
|
||||
fetchLibraryIdsByBookIds: vi.fn(() => Promise.resolve(new Map())),
|
||||
// Batch methods
|
||||
batchUpsertBookMetadata: vi.fn(() => Promise.resolve()),
|
||||
batchDeleteBookAuthors: vi.fn(() => Promise.resolve()),
|
||||
@@ -272,7 +279,9 @@ describe('CoverImporter book cover import', () => {
|
||||
|
||||
const repo = createRepoMock();
|
||||
const importRepo = createImportRepoMock();
|
||||
const importer = new CoverImporter(repo as never, importRepo as never);
|
||||
const scanGateway = createScanGatewayMock();
|
||||
importRepo.fetchLibraryIdsByBookIds.mockResolvedValue(new Map([[901, 1]]));
|
||||
const importer = new CoverImporter(repo as never, importRepo as never, scanGateway as never);
|
||||
|
||||
const planned = {
|
||||
execution: {
|
||||
@@ -294,6 +303,7 @@ describe('CoverImporter book cover import', () => {
|
||||
expect(copiedThumbnailBytes.equals(sourceThumbnailBytes)).toBe(true);
|
||||
|
||||
expect(importRepo.markCoverAsCustom).toHaveBeenCalledWith(901);
|
||||
expect(scanGateway.emitCoverRefreshed).toHaveBeenCalledWith({ bookId: 901, libraryId: 1 });
|
||||
expect(repo.setRunMetric).toHaveBeenCalledWith(
|
||||
52,
|
||||
'book_covers',
|
||||
@@ -316,7 +326,9 @@ describe('CoverImporter book cover import', () => {
|
||||
|
||||
const repo = createRepoMock();
|
||||
const importRepo = createImportRepoMock();
|
||||
const importer = new CoverImporter(repo as never, importRepo as never);
|
||||
const scanGateway = createScanGatewayMock();
|
||||
importRepo.fetchLibraryIdsByBookIds.mockResolvedValue(new Map([[902, 1]]));
|
||||
const importer = new CoverImporter(repo as never, importRepo as never, scanGateway as never);
|
||||
|
||||
const planned = {
|
||||
execution: {
|
||||
@@ -328,6 +340,7 @@ describe('CoverImporter book cover import', () => {
|
||||
|
||||
const generatedThumbnail = await readFile(join(booksPath, 'covers', '902', 'thumbnail.jpg'));
|
||||
expect(generatedThumbnail.length).toBeGreaterThan(0);
|
||||
expect(scanGateway.emitCoverRefreshed).toHaveBeenCalledWith({ bookId: 902, libraryId: 1 });
|
||||
expect(repo.setRunMetric).toHaveBeenCalledWith(
|
||||
53,
|
||||
'book_covers',
|
||||
@@ -343,7 +356,8 @@ describe('CoverImporter book cover import', () => {
|
||||
it('skips book cover stage when source media root is not configured', async () => {
|
||||
const repo = createRepoMock();
|
||||
const importRepo = createImportRepoMock();
|
||||
const importer = new CoverImporter(repo as never, importRepo as never);
|
||||
const scanGateway = createScanGatewayMock();
|
||||
const importer = new CoverImporter(repo as never, importRepo as never, scanGateway as never);
|
||||
|
||||
const planned = {
|
||||
execution: {
|
||||
@@ -354,6 +368,7 @@ describe('CoverImporter book cover import', () => {
|
||||
await importer.import(54, planned as never, '/tmp/books', null, async () => {});
|
||||
|
||||
expect(importRepo.markCoverAsCustom).not.toHaveBeenCalled();
|
||||
expect(scanGateway.emitCoverRefreshed).not.toHaveBeenCalled();
|
||||
expect(repo.setRunMetric).toHaveBeenCalledWith(
|
||||
54,
|
||||
'book_covers',
|
||||
|
||||
@@ -121,9 +121,10 @@ export class MigrationExecutorService {
|
||||
return;
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const cause = error instanceof Error && error.cause instanceof Error ? ` cause="${error.cause.message}"` : '';
|
||||
const errorClass = error instanceof Error ? error.constructor.name : 'UnknownError';
|
||||
this.logger.warn(
|
||||
`[migration.execute] [fail] runId=${runId} durationMs=${durationMs} errorClass=${errorClass} error="${message}" - migration execution failed`,
|
||||
`[migration.execute] [fail] runId=${runId} durationMs=${durationMs} errorClass=${errorClass} error="${message}"${cause} - migration execution failed`,
|
||||
);
|
||||
const failedStage = await this.failRun(runId, message);
|
||||
await this.emitRunProgress(runId, 'failed', failedStage);
|
||||
|
||||
@@ -546,4 +546,20 @@ export class MigrationImportRepository {
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async fetchLibraryIdsByBookIds(bookIds: number[]): Promise<Map<number, number>> {
|
||||
const result = new Map<number, number>();
|
||||
if (bookIds.length === 0) return result;
|
||||
|
||||
const rows = await this.db
|
||||
.select({ id: schema.books.id, libraryId: schema.books.libraryId })
|
||||
.from(schema.books)
|
||||
.where(inArray(schema.books.id, bookIds));
|
||||
|
||||
for (const row of rows) {
|
||||
result.set(row.id, row.libraryId);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { JwtModule } from '@nestjs/jwt';
|
||||
import type { StringValue } from 'ms';
|
||||
|
||||
import { AuthModule } from '../auth/auth.module';
|
||||
import { ScannerModule } from '../scanner/scanner.module';
|
||||
import { MigrationController } from './migration.controller';
|
||||
import { MigrationProgressGateway } from './migration-progress.gateway';
|
||||
import { MigrationRepository } from './migration.repository';
|
||||
@@ -27,6 +28,7 @@ import { MigrationReportingService } from './reporting/migration-reporting.servi
|
||||
@Module({
|
||||
imports: [
|
||||
AuthModule,
|
||||
ScannerModule,
|
||||
JwtModule.registerAsync({
|
||||
inject: [ConfigService],
|
||||
useFactory: (config: ConfigService) => ({
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
|
||||
import type { SourceBook, SourceExportData } from '../adapters/source-adapter.types';
|
||||
import { asRecord } from '../core/coerce';
|
||||
import { MigrationRepository } from '../migration.repository';
|
||||
import { sanitizeRunForApi } from '../core/api-sanitizers';
|
||||
import type { PlannedBookMatch, PlannedDuplicateBookMatch, PlannedUserPreview } from '../planner/planner.types';
|
||||
|
||||
@Injectable()
|
||||
export class MigrationReportingService {
|
||||
@@ -15,12 +18,45 @@ export class MigrationReportingService {
|
||||
this.repo.listRunMetrics(runId),
|
||||
run.planArtifactId ? this.repo.findPlanArtifactById(run.planArtifactId) : Promise.resolve(null),
|
||||
]);
|
||||
const totals = summarizeMetrics(metrics);
|
||||
const sourceData = (artifact?.sourceData as SourceExportData | null) ?? null;
|
||||
const plan = artifact?.plan ?? null;
|
||||
const summary = artifact?.summary ?? null;
|
||||
const sourceBooksById = new Map(sourceData?.books.map((book) => [book.sourceBookId, book]) ?? []);
|
||||
const matchedBooks = normalizeMatchedBooks(asRecord(plan).matchedBooks);
|
||||
const unresolvedBooks = normalizeUnresolvedBooks(asRecord(plan).unresolvedBooks, sourceBooksById);
|
||||
const duplicateBookMatches = normalizeDuplicateBookMatches(asRecord(plan).duplicateBookMatches);
|
||||
const userPreview = normalizeUserPreview(asRecord(plan).userPreview, asRecord(summary).perUserPreview);
|
||||
const targetTitlesById = await this.repo.findBookTitlesByIds(
|
||||
uniqueNumbers([...matchedBooks.map((row) => row.targetBookId), ...duplicateBookMatches.map((row) => row.targetBookId)]),
|
||||
);
|
||||
|
||||
return {
|
||||
run: sanitizeRunForApi(run),
|
||||
totals,
|
||||
metrics,
|
||||
plan: artifact?.plan ?? null,
|
||||
summary: artifact?.summary ?? null,
|
||||
plan,
|
||||
summary,
|
||||
details: {
|
||||
matchedBooks: matchedBooks.map((row) => ({
|
||||
sourceBookId: row.sourceBookId,
|
||||
sourceTitle: sourceBooksById.get(row.sourceBookId)?.title ?? null,
|
||||
sourceAuthor: sourceBooksById.get(row.sourceBookId)?.author ?? null,
|
||||
targetBookId: row.targetBookId,
|
||||
targetTitle: targetTitlesById.get(row.targetBookId) ?? null,
|
||||
strategy: row.strategy,
|
||||
})),
|
||||
unresolvedBooks,
|
||||
duplicateBookMatches: duplicateBookMatches.map((row) => ({
|
||||
targetBookId: row.targetBookId,
|
||||
targetTitle: targetTitlesById.get(row.targetBookId) ?? null,
|
||||
sourceBookIds: row.sourceBookIds,
|
||||
sourceTitles: row.sourceBookIds.map((sourceBookId) => sourceBooksById.get(sourceBookId)?.title ?? null),
|
||||
strategies: row.strategies,
|
||||
reason: row.reason,
|
||||
})),
|
||||
userPreview,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -71,11 +107,22 @@ export class MigrationReportingService {
|
||||
section: 'summary',
|
||||
stage: null,
|
||||
entityType: null,
|
||||
processed: report.metrics.reduce((sum, row) => sum + row.processed, 0),
|
||||
imported: report.metrics.reduce((sum, row) => sum + row.imported, 0),
|
||||
skipped: report.metrics.reduce((sum, row) => sum + row.skipped, 0),
|
||||
unresolved: report.metrics.reduce((sum, row) => sum + row.unresolved, 0),
|
||||
failed: report.metrics.reduce((sum, row) => sum + row.failed, 0),
|
||||
processed: report.totals.processed,
|
||||
imported: report.totals.imported,
|
||||
skipped: report.totals.skipped,
|
||||
unresolved: report.totals.unresolved,
|
||||
failed: report.totals.failed,
|
||||
sourceBookId: null,
|
||||
sourceTitle: null,
|
||||
sourceAuthor: null,
|
||||
targetBookId: null,
|
||||
targetTitle: null,
|
||||
strategy: null,
|
||||
sourceUserId: null,
|
||||
targetUserId: null,
|
||||
username: null,
|
||||
reason: null,
|
||||
details: `matchedBooks=${report.details.matchedBooks.length}; unresolvedBooks=${report.details.unresolvedBooks.length}; mappedUsers=${report.details.userPreview.length}`,
|
||||
code: 'summary',
|
||||
message: `Run state: ${report.run.state}`,
|
||||
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
|
||||
@@ -89,10 +136,117 @@ export class MigrationReportingService {
|
||||
skipped: row.skipped,
|
||||
unresolved: row.unresolved,
|
||||
failed: row.failed,
|
||||
sourceBookId: null,
|
||||
sourceTitle: null,
|
||||
sourceAuthor: null,
|
||||
targetBookId: null,
|
||||
targetTitle: null,
|
||||
strategy: null,
|
||||
sourceUserId: null,
|
||||
targetUserId: null,
|
||||
username: null,
|
||||
reason: null,
|
||||
details: null,
|
||||
code: null,
|
||||
message: null,
|
||||
createdAt: row.updatedAt.toISOString(),
|
||||
})),
|
||||
...report.details.matchedBooks.map((row) => ({
|
||||
section: 'matched_books',
|
||||
stage: 'shared_overlays',
|
||||
entityType: 'book_match',
|
||||
processed: null,
|
||||
imported: null,
|
||||
skipped: null,
|
||||
unresolved: null,
|
||||
failed: null,
|
||||
sourceBookId: row.sourceBookId,
|
||||
sourceTitle: row.sourceTitle,
|
||||
sourceAuthor: row.sourceAuthor,
|
||||
targetBookId: row.targetBookId,
|
||||
targetTitle: row.targetTitle,
|
||||
strategy: row.strategy,
|
||||
sourceUserId: null,
|
||||
targetUserId: null,
|
||||
username: null,
|
||||
reason: null,
|
||||
details: null,
|
||||
code: null,
|
||||
message: null,
|
||||
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
|
||||
})),
|
||||
...report.details.unresolvedBooks.map((row) => ({
|
||||
section: 'unresolved_books',
|
||||
stage: null,
|
||||
entityType: 'book_match',
|
||||
processed: null,
|
||||
imported: null,
|
||||
skipped: null,
|
||||
unresolved: null,
|
||||
failed: null,
|
||||
sourceBookId: row.sourceBookId,
|
||||
sourceTitle: row.title,
|
||||
sourceAuthor: row.author,
|
||||
targetBookId: null,
|
||||
targetTitle: null,
|
||||
strategy: null,
|
||||
sourceUserId: null,
|
||||
targetUserId: null,
|
||||
username: null,
|
||||
reason: row.reason,
|
||||
details: null,
|
||||
code: null,
|
||||
message: null,
|
||||
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
|
||||
})),
|
||||
...report.details.duplicateBookMatches.map((row) => ({
|
||||
section: 'duplicate_matches',
|
||||
stage: null,
|
||||
entityType: 'book_match',
|
||||
processed: null,
|
||||
imported: null,
|
||||
skipped: null,
|
||||
unresolved: null,
|
||||
failed: null,
|
||||
sourceBookId: row.sourceBookIds.join('|'),
|
||||
sourceTitle: row.sourceTitles.filter((title): title is string => typeof title === 'string').join(' | '),
|
||||
sourceAuthor: null,
|
||||
targetBookId: row.targetBookId,
|
||||
targetTitle: row.targetTitle,
|
||||
strategy: row.strategies.join('|'),
|
||||
sourceUserId: null,
|
||||
targetUserId: null,
|
||||
username: null,
|
||||
reason: row.reason,
|
||||
details: `sourceBookIds=${row.sourceBookIds.join('|')}`,
|
||||
code: null,
|
||||
message: null,
|
||||
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
|
||||
})),
|
||||
...report.details.userPreview.map((row) => ({
|
||||
section: 'user_preview',
|
||||
stage: 'user_state',
|
||||
entityType: 'user_preview',
|
||||
processed: null,
|
||||
imported: null,
|
||||
skipped: null,
|
||||
unresolved: null,
|
||||
failed: null,
|
||||
sourceBookId: null,
|
||||
sourceTitle: null,
|
||||
sourceAuthor: null,
|
||||
targetBookId: null,
|
||||
targetTitle: null,
|
||||
strategy: null,
|
||||
sourceUserId: row.sourceUserId,
|
||||
targetUserId: row.targetUserId,
|
||||
username: row.username,
|
||||
reason: null,
|
||||
details: `statuses=${row.counts.statuses}; fileProgress=${row.counts.fileProgress}; bookmarks=${row.counts.bookmarks}; annotations=${row.counts.annotations}; shelves=${row.counts.shelves}`,
|
||||
code: null,
|
||||
message: null,
|
||||
createdAt: report.run.updatedAt?.toISOString() ?? report.run.createdAt.toISOString(),
|
||||
})),
|
||||
];
|
||||
|
||||
return {
|
||||
@@ -123,3 +277,112 @@ function csvCell(value: string | number | null | undefined): string {
|
||||
if (!/[",\n]/.test(text)) return text;
|
||||
return `"${text.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
function summarizeMetrics(metrics: Array<{ processed: number; imported: number; skipped: number; unresolved: number; failed: number }>) {
|
||||
return metrics.reduce(
|
||||
(acc, row) => ({
|
||||
processed: acc.processed + row.processed,
|
||||
imported: acc.imported + row.imported,
|
||||
skipped: acc.skipped + row.skipped,
|
||||
unresolved: acc.unresolved + row.unresolved,
|
||||
failed: acc.failed + row.failed,
|
||||
}),
|
||||
{
|
||||
processed: 0,
|
||||
imported: 0,
|
||||
skipped: 0,
|
||||
unresolved: 0,
|
||||
failed: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeMatchedBooks(value: unknown): PlannedBookMatch[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.flatMap((row) => {
|
||||
const record = asRecord(row);
|
||||
const sourceBookId = typeof record.sourceBookId === 'string' ? record.sourceBookId : null;
|
||||
const targetBookId = typeof record.targetBookId === 'number' ? record.targetBookId : null;
|
||||
const strategy = typeof record.strategy === 'string' ? record.strategy : null;
|
||||
if (!sourceBookId || targetBookId == null || !strategy) return [];
|
||||
return [{ sourceBookId, targetBookId, strategy: strategy as PlannedBookMatch['strategy'] }];
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeUnresolvedBooks(
|
||||
value: unknown,
|
||||
sourceBooksById: Map<string, SourceBook>,
|
||||
): Array<{ sourceBookId: string; title: string | null; author: string | null; reason: string }> {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.flatMap((row) => {
|
||||
const record = asRecord(row);
|
||||
const sourceBookId = typeof record.sourceBookId === 'string' ? record.sourceBookId : null;
|
||||
const reason = typeof record.reason === 'string' ? record.reason : null;
|
||||
if (!sourceBookId || !reason) return [];
|
||||
const sourceBook = sourceBooksById.get(sourceBookId);
|
||||
return [
|
||||
{
|
||||
sourceBookId,
|
||||
title: (typeof record.title === 'string' ? record.title : null) ?? sourceBook?.title ?? null,
|
||||
author: sourceBook?.author ?? null,
|
||||
reason,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeDuplicateBookMatches(value: unknown): PlannedDuplicateBookMatch[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.flatMap((row) => {
|
||||
const record = asRecord(row);
|
||||
const targetBookId = typeof record.targetBookId === 'number' ? record.targetBookId : null;
|
||||
const sourceBookIds = Array.isArray(record.sourceBookIds) ? record.sourceBookIds.filter((id): id is string => typeof id === 'string') : [];
|
||||
const strategies = Array.isArray(record.strategies)
|
||||
? record.strategies.filter((strategy): strategy is PlannedBookMatch['strategy'] => typeof strategy === 'string')
|
||||
: [];
|
||||
if (targetBookId == null || sourceBookIds.length === 0) return [];
|
||||
return [
|
||||
{
|
||||
targetBookId,
|
||||
sourceBookIds,
|
||||
matches: sourceBookIds.map((sourceBookId, index) => ({
|
||||
sourceBookId,
|
||||
targetBookId,
|
||||
strategy: strategies[index] ?? 'title_author',
|
||||
})),
|
||||
strategies,
|
||||
reason: 'duplicate_target_match',
|
||||
} satisfies PlannedDuplicateBookMatch,
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeUserPreview(planValue: unknown, summaryValue: unknown): PlannedUserPreview[] {
|
||||
const rows = Array.isArray(planValue) ? planValue : Array.isArray(summaryValue) ? summaryValue : [];
|
||||
return rows.flatMap((row) => {
|
||||
const record = asRecord(row);
|
||||
const counts = asRecord(record.counts);
|
||||
const sourceUserId = typeof record.sourceUserId === 'string' ? record.sourceUserId : null;
|
||||
const targetUserId = typeof record.targetUserId === 'number' ? record.targetUserId : null;
|
||||
const username = typeof record.username === 'string' ? record.username : null;
|
||||
if (!sourceUserId || targetUserId == null || !username) return [];
|
||||
return [
|
||||
{
|
||||
sourceUserId,
|
||||
targetUserId,
|
||||
username,
|
||||
counts: {
|
||||
statuses: typeof counts.statuses === 'number' ? counts.statuses : 0,
|
||||
fileProgress: typeof counts.fileProgress === 'number' ? counts.fileProgress : 0,
|
||||
bookmarks: typeof counts.bookmarks === 'number' ? counts.bookmarks : 0,
|
||||
annotations: typeof counts.annotations === 'number' ? counts.annotations : 0,
|
||||
shelves: typeof counts.shelves === 'number' ? counts.shelves : 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
function uniqueNumbers(values: number[]): number[] {
|
||||
return [...new Set(values.filter((value) => Number.isFinite(value)))];
|
||||
}
|
||||
|
||||
@@ -29,6 +29,6 @@ import { ScannerService } from './scanner.service';
|
||||
],
|
||||
controllers: [ScannerController],
|
||||
providers: [ScannerService, ScannerRepository, ScanGateway, ScanJobStore, FileEventProcessorService, FileWatcherService],
|
||||
exports: [ScannerService, FileWatcherService],
|
||||
exports: [ScannerService, FileWatcherService, ScanGateway],
|
||||
})
|
||||
export class ScannerModule {}
|
||||
|
||||
@@ -7,6 +7,7 @@ function makeDb(options?: {
|
||||
refreshDeleteResult?: { rowCount?: number };
|
||||
resetDeleteResult?: { rowCount?: number };
|
||||
oidcDeleteResult?: { rowCount?: number };
|
||||
authStateTablesPresent?: boolean;
|
||||
}) {
|
||||
const refreshDeleteBuilder = {
|
||||
where: vi.fn().mockResolvedValue(options?.refreshDeleteResult ?? { rowCount: 2 }),
|
||||
@@ -19,6 +20,12 @@ function makeDb(options?: {
|
||||
};
|
||||
|
||||
return {
|
||||
execute: vi.fn().mockResolvedValue({
|
||||
rows:
|
||||
(options?.authStateTablesPresent ?? true)
|
||||
? [{ table_name: 'refresh_tokens' }, { table_name: 'password_reset_tokens' }, { table_name: 'oidc_sessions' }]
|
||||
: [],
|
||||
}),
|
||||
delete: vi.fn().mockReturnValueOnce(refreshDeleteBuilder).mockReturnValueOnce(resetDeleteBuilder).mockReturnValueOnce(oidcDeleteBuilder),
|
||||
__refreshDeleteBuilder: refreshDeleteBuilder,
|
||||
__resetDeleteBuilder: resetDeleteBuilder,
|
||||
@@ -48,6 +55,7 @@ describe('CleanupService', () => {
|
||||
|
||||
await service.cleanup();
|
||||
|
||||
expect(db.execute).toHaveBeenCalledTimes(1);
|
||||
expect(db.delete).toHaveBeenNthCalledWith(1, schema.refreshTokens);
|
||||
expect(db.delete).toHaveBeenNthCalledWith(2, schema.passwordResetTokens);
|
||||
expect(db.delete).toHaveBeenNthCalledWith(3, schema.oidcSessions);
|
||||
@@ -58,6 +66,18 @@ describe('CleanupService', () => {
|
||||
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[seed.cleanup_auth_state] [end] refreshDeleted=2 resetDeleted=1 oidcDeleted=3'));
|
||||
});
|
||||
|
||||
it('skips cleanup when auth state tables are missing', async () => {
|
||||
const db = makeDb({ authStateTablesPresent: false });
|
||||
const logSpy = vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
|
||||
const service = new CleanupService(db as never);
|
||||
|
||||
await service.cleanup();
|
||||
|
||||
expect(db.execute).toHaveBeenCalledTimes(1);
|
||||
expect(db.delete).not.toHaveBeenCalled();
|
||||
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[seed.cleanup_auth_state] [end] skipped=true reason=auth_tables_missing'));
|
||||
});
|
||||
|
||||
it('logs failure and rethrows on cleanup errors', async () => {
|
||||
const db = makeDb();
|
||||
const err = new Error('delete failed');
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Inject, Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { Cron } from '@nestjs/schedule';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { eq, isNotNull, lt, or } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
@@ -17,12 +18,27 @@ export class CleanupService implements OnApplicationBootstrap {
|
||||
await this.cleanup();
|
||||
}
|
||||
|
||||
private async hasAuthStateTables(): Promise<boolean> {
|
||||
const result = await this.db.execute<{ table_name: string }>(sql`
|
||||
select table_name
|
||||
from information_schema.tables
|
||||
where table_schema = 'public'
|
||||
and table_name in ('refresh_tokens', 'password_reset_tokens', 'oidc_sessions')
|
||||
`);
|
||||
return result.rows.length === 3;
|
||||
}
|
||||
|
||||
@Cron('0 3 * * *')
|
||||
async cleanup() {
|
||||
const startedAt = Date.now();
|
||||
const event = 'seed.cleanup_auth_state';
|
||||
this.logger.log(`[${event}] [start] - auth state cleanup started`);
|
||||
try {
|
||||
if (!(await this.hasAuthStateTables())) {
|
||||
this.logger.log(`[${event}] [end] skipped=true reason=auth_tables_missing durationMs=${Date.now() - startedAt} - auth state cleanup skipped`);
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
const { rowCount: refreshCount } = await this.db
|
||||
|
||||
@@ -272,19 +272,19 @@ export async function seedCanonicalScenario(ctx: MigrationBookloreE2EContext): P
|
||||
null,
|
||||
'Hash description',
|
||||
null,
|
||||
101,
|
||||
null,
|
||||
-5,
|
||||
null,
|
||||
null,
|
||||
12,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
-30,
|
||||
0,
|
||||
null,
|
||||
],
|
||||
|
||||
@@ -216,9 +216,11 @@ export async function waitForMigrationToFinish(
|
||||
};
|
||||
report: {
|
||||
run: { id: number; state: string; currentStage: string | null };
|
||||
totals: { processed: number; imported: number; skipped: number; unresolved: number; failed: number };
|
||||
metrics: Array<Record<string, unknown>>;
|
||||
plan: Record<string, unknown> | null;
|
||||
summary: Record<string, unknown> | null;
|
||||
details: Record<string, unknown>;
|
||||
};
|
||||
}> {
|
||||
let latestProgress:
|
||||
@@ -255,9 +257,11 @@ export async function waitForMigrationToFinish(
|
||||
|
||||
const reportResponse = await apiJson<{
|
||||
run: { id: number; state: string; currentStage: string | null };
|
||||
totals: { processed: number; imported: number; skipped: number; unresolved: number; failed: number };
|
||||
metrics: Array<Record<string, unknown>>;
|
||||
plan: Record<string, unknown> | null;
|
||||
summary: Record<string, unknown> | null;
|
||||
details: Record<string, unknown>;
|
||||
}>(ctx, {
|
||||
method: 'GET',
|
||||
url: `/api/v1/migration/runs/${runId}/report`,
|
||||
|
||||
@@ -369,7 +369,25 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
|
||||
});
|
||||
expect(jsonExport.statusCode).toBe(200);
|
||||
expect(jsonExport.headers['content-type']).toContain('application/json');
|
||||
expect(jsonExport.body).toContain('"state": "completed"');
|
||||
const parsedJsonExport = JSON.parse(jsonExport.body) as {
|
||||
run: { state: string };
|
||||
details: {
|
||||
matchedBooks: Array<{ sourceBookId: string; sourceTitle: string | null; targetBookId: number; strategy: string }>;
|
||||
userPreview: unknown[];
|
||||
};
|
||||
};
|
||||
expect(parsedJsonExport.run.state).toBe('completed');
|
||||
expect(parsedJsonExport.details.matchedBooks).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
sourceBookId: '101',
|
||||
sourceTitle: 'Imported ISBN Title',
|
||||
targetBookId: scenario.books.isbn.bookId,
|
||||
strategy: 'isbn',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(parsedJsonExport.details.userPreview).toHaveLength(2);
|
||||
|
||||
const csvExport = await apiText(ctx, {
|
||||
method: 'GET',
|
||||
@@ -378,7 +396,44 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
|
||||
});
|
||||
expect(csvExport.statusCode).toBe(200);
|
||||
expect(csvExport.headers['content-type']).toContain('text/csv');
|
||||
expect(csvExport.body).toContain('section,stage,entityType,processed,imported,skipped,unresolved,failed,code,message,createdAt');
|
||||
expect(csvExport.body).toContain(
|
||||
'section,stage,entityType,processed,imported,skipped,unresolved,failed,sourceBookId,sourceTitle,sourceAuthor,targetBookId,targetTitle,strategy,sourceUserId,targetUserId,username,reason,details,code,message,createdAt',
|
||||
);
|
||||
expect(csvExport.body).toContain('matched_books,shared_overlays,book_match');
|
||||
expect(csvExport.body).toContain('Imported ISBN Title');
|
||||
|
||||
const reportDetails = finished.report.details as {
|
||||
matchedBooks?: Array<{ sourceBookId: string; sourceTitle: string | null; targetBookId: number; strategy: string }>;
|
||||
userPreview?: Array<{ username: string; counts: Record<string, number> }>;
|
||||
};
|
||||
expect(reportDetails.matchedBooks).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
sourceBookId: '101',
|
||||
sourceTitle: 'Imported ISBN Title',
|
||||
targetBookId: scenario.books.isbn.bookId,
|
||||
strategy: 'isbn',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
sourceBookId: '102',
|
||||
sourceTitle: 'Imported Hash Title',
|
||||
targetBookId: scenario.books.hash.bookId,
|
||||
strategy: 'file_hash',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(reportDetails.userPreview).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
username: 'alice',
|
||||
counts: expect.objectContaining({ statuses: 2, fileProgress: 2, bookmarks: 1, annotations: 1, shelves: 2 }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
username: 'bob',
|
||||
counts: expect.objectContaining({ statuses: 1, fileProgress: 1, bookmarks: 1, annotations: 1, shelves: 1 }),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const metricByKey = buildMetricMap(finished.report.metrics as MetricRow[]);
|
||||
expect(metricByKey.get('shared_overlays:book_metadata')).toMatchObject({
|
||||
@@ -478,8 +533,11 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
|
||||
title: 'Imported Hash Title',
|
||||
description: 'Hash description',
|
||||
subtitle: null,
|
||||
publishedYear: null,
|
||||
pageCount: null,
|
||||
rating: null,
|
||||
googleBooksId: null,
|
||||
durationSeconds: null,
|
||||
});
|
||||
expect(metadataByBookId.get(scenario.books.audio.bookId)).toMatchObject({
|
||||
title: 'Audio Path Match',
|
||||
|
||||
Reference in New Issue
Block a user