From 9fd4e162aaba4955e0d70beeb1e40d55dde0e6d4 Mon Sep 17 00:00:00 2001 From: neon <18559658+neonsolstice@users.noreply.github.com> Date: Sun, 5 Apr 2026 16:01:04 -0600 Subject: [PATCH] feat(server): harden booklore migration with adapters and E2E tests --- .github/workflows/ci.yml | 8 + .../migration/components/MigrationModal.vue | 1559 +++++++++++++++++ .../migration/components/MigrationStepper.vue | 63 + .../components/SearchableUserSelect.vue | 97 + .../composables/useMigrationPolling.ts | 51 + .../composables/useMigrationProgress.ts | 57 + .../features/migration/lib/migration-api.ts | 128 +- .../features/settings/MaintenanceSettings.vue | 181 +- .../features/settings/MigrationSettings.vue | 1335 ++++++++------ .../src/features/settings/SettingsHeader.vue | 3 - client/src/router/index.ts | 6 - packages/types/src/index.ts | 1 + packages/types/src/migration.ts | 106 ++ scripts/test/e2e-suites.mjs | 10 + server/.env.example | 5 + server/src/app.module.ts | 4 +- server/src/config/config.ts | 4 + .../{0000_baseline.sql => 0000_initial.sql} | 5 +- .../src/db/migrations/meta/0000_snapshot.json | 30 +- server/src/db/migrations/meta/_journal.json | 4 +- server/src/db/schema/migration.ts | 14 +- .../architecture-boundaries.test.ts | 1 - .../booklore/booklore-connector.test.ts | 25 + .../adapters/booklore/booklore-connector.ts | 7 + .../booklore/booklore-source.adapter.test.ts | 57 + .../booklore/booklore-source.adapter.ts | 137 +- .../adapters/booklore/booklore-tables.ts | 36 + .../adapters/source-adapter.types.ts | 15 + .../migration/core/api-sanitizers.test.ts | 161 ++ .../modules/migration/core/api-sanitizers.ts | 116 ++ .../src/modules/migration/core/coerce.test.ts | 137 ++ server/src/modules/migration/core/coerce.ts | 25 + .../migration/core/connection-config.test.ts | 44 + .../migration/core/connection-config.ts | 11 + .../core/migration-encryption.service.test.ts | 116 ++ .../core/migration-encryption.service.ts | 75 + .../src/modules/migration/core/plan-hash.ts | 4 + .../dto/resolve-duplicate-matches.dto.ts | 19 + .../migration/executor/cover.importer.ts | 131 ++ .../migration/executor/executor-utils.ts | 247 +++ .../migration-executor.service.test.ts | 236 +-- .../executor/migration-executor.service.ts | 1168 ++---------- .../executor/migration-import.repository.ts | 549 ++++++ .../executor/shared-overlays.importer.ts | 261 +++ .../migration/executor/user-state.importer.ts | 522 ++++++ .../migration-profile.service.test.ts | 172 ++ .../migration/migration-profile.service.ts | 178 ++ .../migration/migration-progress.gateway.ts | 95 + .../migration-source.service.test.ts | 193 ++ .../migration/migration-source.service.ts | 161 ++ .../modules/migration/migration.controller.ts | 66 +- .../migration/migration.module.test.ts | 8 + .../src/modules/migration/migration.module.ts | 30 + .../modules/migration/migration.repository.ts | 161 +- .../migration/migration.service.test.ts | 521 +++++- .../modules/migration/migration.service.ts | 611 +++---- .../planner/matching.service.test.ts | 25 + .../migration/planner/matching.service.ts | 305 +++- .../path-mapping-validation.service.ts | 3 +- .../migration/planner/planner.service.ts | 43 +- .../migration/planner/planner.types.ts | 8 +- .../reporting/migration-reporting.service.ts | 17 +- server/test/e2e/app-harness.ts | 10 + .../migration-booklore-fixture-builder.ts | 1030 +++++++++++ .../migration-booklore-harness.ts | 521 ++++++ server/test/migration-booklore.e2e-spec.ts | 1060 +++++++++++ 66 files changed, 10540 insertions(+), 2449 deletions(-) create mode 100644 client/src/features/migration/components/MigrationModal.vue create mode 100644 client/src/features/migration/components/MigrationStepper.vue create mode 100644 client/src/features/migration/components/SearchableUserSelect.vue create mode 100644 client/src/features/migration/composables/useMigrationPolling.ts create mode 100644 client/src/features/migration/composables/useMigrationProgress.ts create mode 100644 packages/types/src/migration.ts rename server/src/db/migrations/{0000_baseline.sql => 0000_initial.sql} (99%) create mode 100644 server/src/modules/migration/adapters/booklore/booklore-connector.test.ts create mode 100644 server/src/modules/migration/adapters/booklore/booklore-tables.ts create mode 100644 server/src/modules/migration/core/api-sanitizers.test.ts create mode 100644 server/src/modules/migration/core/api-sanitizers.ts create mode 100644 server/src/modules/migration/core/coerce.test.ts create mode 100644 server/src/modules/migration/core/coerce.ts create mode 100644 server/src/modules/migration/core/connection-config.test.ts create mode 100644 server/src/modules/migration/core/connection-config.ts create mode 100644 server/src/modules/migration/core/migration-encryption.service.test.ts create mode 100644 server/src/modules/migration/core/migration-encryption.service.ts create mode 100644 server/src/modules/migration/dto/resolve-duplicate-matches.dto.ts create mode 100644 server/src/modules/migration/executor/cover.importer.ts create mode 100644 server/src/modules/migration/executor/executor-utils.ts create mode 100644 server/src/modules/migration/executor/migration-import.repository.ts create mode 100644 server/src/modules/migration/executor/shared-overlays.importer.ts create mode 100644 server/src/modules/migration/executor/user-state.importer.ts create mode 100644 server/src/modules/migration/migration-profile.service.test.ts create mode 100644 server/src/modules/migration/migration-profile.service.ts create mode 100644 server/src/modules/migration/migration-progress.gateway.ts create mode 100644 server/src/modules/migration/migration-source.service.test.ts create mode 100644 server/src/modules/migration/migration-source.service.ts create mode 100644 server/test/e2e/migration-booklore/migration-booklore-fixture-builder.ts create mode 100644 server/test/e2e/migration-booklore/migration-booklore-harness.ts create mode 100644 server/test/migration-booklore.e2e-spec.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 354aeeebe..538106c6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,6 +340,14 @@ jobs: - 'scripts/test/**' - 'scripts/db/**' - '.github/workflows/e2e-runner.yml' + migration-booklore: + - 'server/src/modules/migration/**' + - 'server/test/migration-booklore.e2e-spec.ts' + - 'server/test/e2e/migration-booklore/**' + - 'server/src/db/schema/**' + - 'scripts/test/**' + - 'scripts/db/**' + - '.github/workflows/e2e-runner.yml' opds-auth-catalog: - 'server/src/modules/opds/**' - 'server/src/modules/book/**' diff --git a/client/src/features/migration/components/MigrationModal.vue b/client/src/features/migration/components/MigrationModal.vue new file mode 100644 index 000000000..643d4fbc2 --- /dev/null +++ b/client/src/features/migration/components/MigrationModal.vue @@ -0,0 +1,1559 @@ + + + diff --git a/client/src/features/migration/components/MigrationStepper.vue b/client/src/features/migration/components/MigrationStepper.vue new file mode 100644 index 000000000..cb79ac4e7 --- /dev/null +++ b/client/src/features/migration/components/MigrationStepper.vue @@ -0,0 +1,63 @@ + + + diff --git a/client/src/features/migration/components/SearchableUserSelect.vue b/client/src/features/migration/components/SearchableUserSelect.vue new file mode 100644 index 000000000..ea56c32fe --- /dev/null +++ b/client/src/features/migration/components/SearchableUserSelect.vue @@ -0,0 +1,97 @@ + + + diff --git a/client/src/features/migration/composables/useMigrationPolling.ts b/client/src/features/migration/composables/useMigrationPolling.ts new file mode 100644 index 000000000..6931bb798 --- /dev/null +++ b/client/src/features/migration/composables/useMigrationPolling.ts @@ -0,0 +1,51 @@ +import { onBeforeUnmount, ref, watch, type Ref } from 'vue' + +interface UseMigrationPollingOptions { + runState: Ref + pollFn: () => Promise + intervalMs?: number +} + +export function useMigrationPolling({ runState, pollFn, intervalMs = 3000 }: UseMigrationPollingOptions) { + let timer: ReturnType | null = null + const pollingError = ref(false) + + function start() { + stop() + timer = setInterval(() => void doPoll(), intervalMs) + } + + function stop() { + if (timer == null) return + clearInterval(timer) + timer = null + } + + async function doPoll() { + try { + await pollFn() + pollingError.value = false + } catch { + stop() + pollingError.value = true + } + } + + function retry() { + pollingError.value = false + start() + void doPoll() + } + + watch(runState, (state) => { + if (state === 'running') { + start() + } else { + stop() + } + }) + + onBeforeUnmount(stop) + + return { pollingError, start, stop, retry } +} diff --git a/client/src/features/migration/composables/useMigrationProgress.ts b/client/src/features/migration/composables/useMigrationProgress.ts new file mode 100644 index 000000000..ab49b7a45 --- /dev/null +++ b/client/src/features/migration/composables/useMigrationProgress.ts @@ -0,0 +1,57 @@ +import { ref } from 'vue' +import { io, Socket } from 'socket.io-client' +import type { MigrationProgressEvent } from '@projectx/types' +import { getAccessToken } from '@/lib/api' + +let socket: Socket | null = null +const progressMap = ref>(new Map()) +const subscribedRuns = new Set() + +function getSocket(): Socket { + if (!socket) { + socket = io('/migration', { + auth: (cb: (data: object) => void) => cb({ token: getAccessToken() }), + transports: ['websocket'], + autoConnect: true, + }) + + socket.on('migration:progress', (event: MigrationProgressEvent) => { + progressMap.value.set(event.runId, event) + progressMap.value = new Map(progressMap.value) + }) + + socket.on('disconnect', () => { + progressMap.value = new Map() + }) + + socket.on('connect', () => { + for (const runId of subscribedRuns) { + socket!.emit('subscribe:run', runId) + } + }) + } + return socket +} + +export function useMigrationProgress() { + function subscribeRun(runId: number): void { + subscribedRuns.add(runId) + getSocket().emit('subscribe:run', runId) + } + + function unsubscribeRun(runId: number): void { + subscribedRuns.delete(runId) + progressMap.value.delete(runId) + progressMap.value = new Map(progressMap.value) + } + + function getProgress(runId: number): MigrationProgressEvent | undefined { + return progressMap.value.get(runId) + } + + function isRunning(runId: number): boolean { + return progressMap.value.get(runId)?.state === 'running' + } + + return { subscribeRun, unsubscribeRun, getProgress, isRunning, progressMap } +} diff --git a/client/src/features/migration/lib/migration-api.ts b/client/src/features/migration/lib/migration-api.ts index 961856938..7fb72e767 100644 --- a/client/src/features/migration/lib/migration-api.ts +++ b/client/src/features/migration/lib/migration-api.ts @@ -1,82 +1,26 @@ import { api } from '@/lib/api' +import type { + MigrationRunState as SharedMigrationRunState, + MigrationSourceApi, + MigrationSourceCapabilities as SharedMigrationSourceCapabilities, + MigrationProfileApi, + MigrationPlanArtifactApi, + MigrationPlanSummary, + MigrationRunApi, + MigrationRunMetricApi, +} from '@projectx/types' -export type MigrationRunState = 'draft' | 'preflight_failed' | 'dry_run_ready' | 'running' | 'failed' | 'completed' +export type MigrationRunState = SharedMigrationRunState +export type MigrationSourceCapabilities = SharedMigrationSourceCapabilities +export type MigrationSource = MigrationSourceApi +export type MigrationProfile = MigrationProfileApi +export type MigrationPlanArtifact = MigrationPlanArtifactApi & { summary: MigrationPlanSummary } +export type MigrationRun = MigrationRunApi -export interface MigrationSourceCapabilities { - sourceType: string - sourceVersion: string | null - missingTables: string[] - warnings: string[] - counts: Record -} - -export interface MigrationSource { - id: number - type: string - name: string - connectionConfig: Record - capabilities: MigrationSourceCapabilities | null - lastValidatedAt: string | null - createdAt: string - updatedAt: string -} - -export interface MigrationProfile { - id: number - sourceId: number - name: string - userMappings: Array<{ sourceUserId: string; targetUserId: number }> - pathMappings: Array<{ sourcePrefix: string; targetPrefix: string }> - scope: Record - createdAt: string - updatedAt: string -} - -export interface MigrationPlanArtifact { - id: number - sourceId: number - profileId: number - plan: Record - summary: { - generatedAt?: string - status?: string - matchedBooks?: number - unresolvedBooks?: number - duplicateBookMatches?: number - unresolvedByReason?: Record - mappedUsers?: number - perUserPreview?: Array<{ sourceUserId: string; targetUserId: number; username: string; counts: Record }> - } - createdAt: string - updatedAt: string -} - -export interface MigrationRun { - id: number - sourceId: number - profileId: number - planArtifactId: number | null - state: MigrationRunState - currentStage: string | null - startedAt: string | null - endedAt: string | null - errorMessage: string | null - createdAt: string - updatedAt: string -} - -export interface MigrationRunMetric { +export interface MigrationRunMetric extends MigrationRunMetricApi { id: number runId: number - stage: string - entityType: string - processed: number - imported: number - skipped: number - unresolved: number - failed: number createdAt: string - updatedAt: string } export interface MigrationRunProgress { @@ -95,7 +39,7 @@ export interface MigrationRunReport { run: MigrationRun metrics: MigrationRunMetric[] plan: Record | null - summary: MigrationPlanArtifact['summary'] | null + summary: MigrationPlanSummary | null } export interface MigrationWorkflowState { @@ -111,6 +55,8 @@ export interface MigrationWorkflowState { export interface PathMappingValidation { sourceId: number validatedAt: string + pathMappingsHash: string + persistedProfileId: number | null summary: { totalSourceBooks: number booksWithFilePath: number @@ -243,6 +189,14 @@ export function createDryRunPlan(payload: { profileId: number; scopeOverride?: R }).then((res) => expectJson(res, 'Failed to run dry-run plan')) } +export function resolveDuplicateMatches(artifactId: number, resolutions: Array<{ targetBookId: number; selectedSourceBookId: string }>) { + return api(`/api/v1/migration/plans/${artifactId}/resolve-duplicates`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ resolutions }), + }).then((res) => expectJson(res, 'Failed to resolve duplicate matches')) +} + export function startLiveRun(payload: { planArtifactId: number; targetKey?: string }) { return api('/api/v1/migration/runs/live', { method: 'POST', @@ -259,8 +213,28 @@ export function getRunReport(runId: number) { return api(`/api/v1/migration/runs/${runId}/report`).then((res) => expectJson(res, 'Failed to load migration run report')) } -export function exportRunReport(runId: number, format: 'json' | 'csv') { - return api(`/api/v1/migration/runs/${runId}/report/export?format=${format}`).then((res) => - expectJson<{ format: 'json' | 'csv'; fileName: string; contentType: string; content: string }>(res, 'Failed to export migration report'), +export async function exportRunReport(runId: number, format: 'json' | 'csv') { + const res = await api(`/api/v1/migration/runs/${runId}/report/export?format=${format}`) + if (!res.ok) { + const body = await res.text().catch(() => '') + throw new Error(body || 'Failed to export migration report') + } + const disposition = res.headers.get('Content-Disposition') ?? '' + const fileNameMatch = disposition.match(/filename="?([^"]+)"?/) + const fileName = fileNameMatch?.[1] ?? `migration-run-${runId}-report.${format}` + const contentType = res.headers.get('Content-Type') ?? (format === 'csv' ? 'text/csv' : 'application/json') + const content = await res.text() + return { format, fileName, contentType, content } +} + +export function cancelRun(runId: number) { + return api(`/api/v1/migration/runs/${runId}/cancel`, { method: 'POST' }).then((res) => + expectJson(res, 'Failed to cancel migration run'), + ) +} + +export function retryRun(runId: number) { + return api(`/api/v1/migration/runs/${runId}/retry`, { method: 'POST' }).then((res) => + expectJson(res, 'Failed to retry migration run'), ) } diff --git a/client/src/features/settings/MaintenanceSettings.vue b/client/src/features/settings/MaintenanceSettings.vue index 69e4bf634..6bcbe8b2c 100644 --- a/client/src/features/settings/MaintenanceSettings.vue +++ b/client/src/features/settings/MaintenanceSettings.vue @@ -1,18 +1,58 @@