From 0c195165e6d57fe361ac89d066b5a38532e4e06a Mon Sep 17 00:00:00 2001 From: neon <18559658+neonsolstice@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:17:27 -0600 Subject: [PATCH] test(scanner): add staging ingest and finalize E2E test suite --- .../workflows/e2e-staging-ingest-finalize.yml | 42 ++ DEVELOPMENT.md | 20 +- client/src/components/AppSidebar.vue | 2 +- scripts/test/e2e-suites.mjs | 8 + .../staging/staging-finalize.service.ts | 21 +- .../e2e/staging/staging-fixture-builder.ts | 93 +++ server/test/e2e/staging/staging-harness.ts | 395 +++++++++++ .../test/staging-ingest-finalize.e2e-spec.ts | 658 ++++++++++++++++++ 8 files changed, 1231 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/e2e-staging-ingest-finalize.yml create mode 100644 server/test/e2e/staging/staging-fixture-builder.ts create mode 100644 server/test/e2e/staging/staging-harness.ts create mode 100644 server/test/staging-ingest-finalize.e2e-spec.ts diff --git a/.github/workflows/e2e-staging-ingest-finalize.yml b/.github/workflows/e2e-staging-ingest-finalize.yml new file mode 100644 index 000000000..78e7f2533 --- /dev/null +++ b/.github/workflows/e2e-staging-ingest-finalize.yml @@ -0,0 +1,42 @@ +name: Staging Ingest Finalize E2E +run-name: ${{ format('Staging Ingest Finalize E2E - {0}', github.ref_name) }} + +on: + workflow_dispatch: + push: + paths: + - "server/src/modules/staging/**" + - "server/src/modules/upload/**" + - "server/src/modules/metadata/**" + - "server/test/staging-ingest-finalize.e2e-spec.ts" + - "server/test/e2e/staging/**" + - "server/src/db/schema/**" + - "scripts/test/**" + - "scripts/db/**" + pull_request: + paths: + - "server/src/modules/staging/**" + - "server/src/modules/upload/**" + - "server/src/modules/metadata/**" + - "server/test/staging-ingest-finalize.e2e-spec.ts" + - "server/test/e2e/staging/**" + - "server/src/db/schema/**" + - "scripts/test/**" + - "scripts/db/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + checks: write + +jobs: + staging-ingest-finalize-e2e: + uses: ./.github/workflows/e2e-runner.yml + with: + suite-id: staging-ingest-finalize + check-name: Server Staging Ingest Finalize E2E + artifact-name: staging-ingest-finalize-e2e-report + job-timeout-minutes: 45 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 54783df65..64e222e74 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -118,6 +118,7 @@ pnpm run test:e2e -- all # run all e2e suites sequentially pnpm run test:e2e -- smoke # e2e smoke suite (default DB) pnpm run test:e2e -- scanner # scanner e2e suite (dedicated projectx_e2e DB) pnpm run test:e2e -- scanner-file-ops # scanner file operation e2e suite (dedicated projectx_e2e DB) +pnpm run test:e2e -- staging-ingest-finalize # staging ingest/finalize e2e suite (dedicated projectx_e2e DB) pnpm run test:e2e:all # alias for "pnpm run test:e2e -- all" pnpm run test:e2e:list # list all supported e2e suite ids ``` @@ -158,6 +159,14 @@ pnpm run test:e2e -- scanner --testNamePattern=book-per-folder-disc-folder-flatt - `scanner-file-ops-e2e-junit.xml` - `scanner-file-ops-e2e-scenarios.json` +### Staging ingest/finalize e2e details + +- `pnpm run test:e2e -- staging-ingest-finalize` prepares and migrates the dedicated e2e database (`projectx_e2e`) before running. +- Local runs auto-start PostgreSQL with `docker-compose.dev.yml` if needed. +- Results are written to `test-results/server/`: + - `staging-ingest-finalize-e2e-junit.xml` + - `staging-ingest-finalize-e2e-scenarios.json` + ### E2E in CI (how to trigger) Both workflows call the reusable `E2E Runner (reusable)` workflow, which runs: @@ -166,15 +175,16 @@ Both workflows call the reusable `E2E Runner (reusable)` workflow, which runs: pnpm run test:e2e -- ``` -| Workflow | Suite id | Triggered by | -| -------------------- | ------------------ | -------------------------------------------------------------------------------- | -| Scanner E2E | `scanner` | `workflow_dispatch`, `push` (scanner-related paths), `pull_request` (same paths) | -| Scanner File Ops E2E | `scanner-file-ops` | `workflow_dispatch`, nightly schedule (`0 4 * * *`), `push`, `pull_request` | +| Workflow | Suite id | Triggered by | +| --------------------------- | ------------------------- | -------------------------------------------------------------------------------- | +| Scanner E2E | `scanner` | `workflow_dispatch`, `push` (scanner-related paths), `pull_request` (same paths) | +| Scanner File Ops E2E | `scanner-file-ops` | `workflow_dispatch`, nightly schedule (`0 4 * * *`), `push`, `pull_request` | +| Staging Ingest Finalize E2E | `staging-ingest-finalize` | `workflow_dispatch`, `push` (staging-related paths), `pull_request` (same paths) | Manual trigger steps: 1. Open GitHub Actions. -2. Select **Scanner E2E** or **Scanner File Ops E2E**. +2. Select **Scanner E2E**, **Scanner File Ops E2E**, or **Staging Ingest Finalize E2E**. 3. Click **Run workflow**. Each run uploads `test-results/server/` as an artifact and publishes JUnit annotations from `test-results/server/*-e2e-junit.xml`. diff --git a/client/src/components/AppSidebar.vue b/client/src/components/AppSidebar.vue index 0d77ea8e4..3479806e6 100644 --- a/client/src/components/AppSidebar.vue +++ b/client/src/components/AppSidebar.vue @@ -171,7 +171,7 @@ onUnmounted(() => stopUploadCompletedListener())
- Project X + Project Q Library Control
diff --git a/scripts/test/e2e-suites.mjs b/scripts/test/e2e-suites.mjs index 78ced4f2e..221f94a49 100644 --- a/scripts/test/e2e-suites.mjs +++ b/scripts/test/e2e-suites.mjs @@ -41,6 +41,14 @@ export const E2E_SUITES = Object.freeze({ prepareDedicatedDatabase: true, useDedicatedDatabase: true, }, + "staging-ingest-finalize": { + id: "staging-ingest-finalize", + description: "Staging ingest and finalize suite", + vitestTarget: "test/staging-ingest-finalize.e2e-spec.ts", + junitOutput: `${TEST_RESULTS_DIR}/staging-ingest-finalize-e2e-junit.xml`, + prepareDedicatedDatabase: true, + useDedicatedDatabase: true, + }, }); export function listE2ESuites() { diff --git a/server/src/modules/staging/staging-finalize.service.ts b/server/src/modules/staging/staging-finalize.service.ts index dd70526e8..66fc322b1 100644 --- a/server/src/modules/staging/staging-finalize.service.ts +++ b/server/src/modules/staging/staging-finalize.service.ts @@ -76,7 +76,7 @@ export class StagingFinalizeService implements OnModuleInit { status?: string, search?: string, ): Promise { - const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : (fileIds ?? []); + const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : dedupeIds(fileIds ?? []); const overrideMap = new Map((overrides ?? []).map((o) => [o.fileId, o])); const results: StagingFinalizeFileResult[] = []; @@ -86,8 +86,21 @@ export class StagingFinalizeService implements OnModuleInit { for (let i = 0; i < ids.length; i += BATCH_SIZE) { const batch = ids.slice(i, i + BATCH_SIZE); const rows = await this.repo.findByIds(batch); + const rowById = new Map(rows.map((row) => [row.id, row])); + + for (const fileId of batch) { + const row = rowById.get(fileId); + if (!row) { + failed++; + results.push({ + fileId, + fileName: `staging-file-${fileId}`, + success: false, + message: 'Staging file not found', + }); + continue; + } - for (const row of rows) { const result = await this.finalizeFile(row, defaultLibraryId, defaultFolderId, overrideMap, userId, isSuperuser); results.push(result); if (result.success) succeeded++; @@ -500,3 +513,7 @@ function shouldAutoFinalize(row: StagingFileRow, mode: StagingAutoFinalizeMetada } return row.confidence !== null && row.confidence >= threshold; } + +function dedupeIds(ids: number[]): number[] { + return [...new Set(ids)]; +} diff --git a/server/test/e2e/staging/staging-fixture-builder.ts b/server/test/e2e/staging/staging-fixture-builder.ts new file mode 100644 index 000000000..02906b025 --- /dev/null +++ b/server/test/e2e/staging/staging-fixture-builder.ts @@ -0,0 +1,93 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { dirname, join } from 'path'; + +export interface StagingFixtureRoot { + rootPath: string; + booksPath: string; + stagingPath: string; + cleanup: () => Promise; +} + +function assertRelativePath(path: string): void { + if (path.startsWith('/')) { + throw new Error(`Fixture paths must be relative. Received "${path}"`); + } +} + +export async function createStagingFixtureRoot(prefix = 'staging-e2e-'): Promise { + const rootPath = await mkdtemp(join(tmpdir(), prefix)); + const booksPath = join(rootPath, 'books'); + const stagingPath = join(booksPath, 'staging'); + + await mkdir(stagingPath, { recursive: true }); + + return { + rootPath, + booksPath, + stagingPath, + cleanup: async () => { + await rm(rootPath, { recursive: true, force: true }); + }, + }; +} + +export async function writeFixtureFile(rootPath: string, relativePath: string, content: string | Buffer): Promise { + assertRelativePath(relativePath); + + const absolutePath = join(rootPath, relativePath); + await mkdir(dirname(absolutePath), { recursive: true }); + await writeFile(absolutePath, content); + return absolutePath; +} + +export interface Fb2FixtureInput { + title?: string; + authors?: string[]; + language?: string; + genre?: string; + description?: string; + year?: number; +} + +export function buildFb2Fixture(input: Fb2FixtureInput = {}): string { + const title = input.title ?? 'Staging Fixture Title'; + const authors = input.authors ?? ['Staging Fixture Author']; + const language = input.language ?? 'en'; + const genre = input.genre ?? 'fiction'; + const description = input.description ?? 'Fixture description'; + const year = input.year ?? 2024; + + const authorXml = authors + .map((name) => { + const [firstName = 'Unknown', ...rest] = name.trim().split(/\s+/); + const lastName = rest.length > 0 ? rest.join(' ') : 'Author'; + return `${escapeXml(firstName)}${escapeXml(lastName)}`; + }) + .join(''); + + return ` + + + + ${escapeXml(genre)} + ${authorXml} + ${escapeXml(title)} +

${escapeXml(description)}

 + ${year} + ${escapeXml(language)} + + + ${year} + + + +

fixture content

+ + +`; +} + +function escapeXml(value: string): string { + return value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"').replaceAll("'", '''); +} diff --git a/server/test/e2e/staging/staging-harness.ts b/server/test/e2e/staging/staging-harness.ts new file mode 100644 index 000000000..69d032ab7 --- /dev/null +++ b/server/test/e2e/staging/staging-harness.ts @@ -0,0 +1,395 @@ +import { randomUUID } from 'crypto'; +import { hash } from 'bcryptjs'; +import { extname, join } from 'path'; +import { mkdir, rm, stat } from 'fs/promises'; + +import fastifyCookie from '@fastify/cookie'; +import fastifyMultipart from '@fastify/multipart'; +import { ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import { FastifyAdapter, type NestFastifyApplication } from '@nestjs/platform-fastify'; +import { eq } from 'drizzle-orm'; +import { Permission, type StagingMetadata } from '@projectx/types'; + +import { AppModule } from '../../../src/app.module'; +import { DB } from '../../../src/db'; +import * as schema from '../../../src/db/schema'; +import { StagingWatcherService } from '../../../src/modules/staging/staging-watcher.service'; +import { seedLibrary, waitForCondition, type Db } from '../app-harness'; +import { buildFb2Fixture, createStagingFixtureRoot, writeFixtureFile, type StagingFixtureRoot } from './staging-fixture-builder'; + +const ADMIN_SETUP_DTO = { + username: 'staging-e2e-admin', + name: 'Staging E2E Admin', + email: 'staging-e2e-admin@example.com', + password: 'StagingAdmin123', +}; + +interface EnvSnapshot { + booksPath: string | undefined; + stagingPath: string | undefined; +} + +export interface StagingE2EContext { + app: NestFastifyApplication; + db: Db; + adminToken: string; + fixture: StagingFixtureRoot; + envSnapshot: EnvSnapshot; +} + +export interface TestUserSession { + userId: number; + username: string; + password: string; + accessToken: string; +} + +export interface UploadStagingFileInput { + token: string; + fileName: string; + content: string | Buffer; + contentType?: string; +} + +export interface CreateStagingRowInput { + fileName?: string; + content?: string | Buffer; + status?: typeof schema.stagingFiles.$inferInsert.status; + format?: string; + embeddedMetadata?: StagingMetadata | null; + selectedMetadata?: StagingMetadata | null; + fetchedMetadata?: StagingMetadata | null; + targetLibraryId?: number | null; + targetFolderId?: number | null; + confidence?: number | null; + errorMessage?: string | null; + metadataEditedAt?: Date | null; +} + +export interface CreatedLibrary { + libraryId: number; + libraryFolderId: number; + folderPath: string; +} + +export function authHeader(token: string): Record { + return { authorization: `Bearer ${token}` }; +} + +export async function createStagingE2EContext(): Promise { + const fixture = await createStagingFixtureRoot(); + const envSnapshot: EnvSnapshot = { + booksPath: process.env.BOOKS_PATH, + stagingPath: process.env.STAGING_PATH, + }; + + process.env.BOOKS_PATH = fixture.booksPath; + process.env.STAGING_PATH = fixture.stagingPath; + + const moduleFixture = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + + const app = moduleFixture.createNestApplication(new FastifyAdapter()); + app.setGlobalPrefix('api/v1'); + app.useGlobalPipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + transform: true, + }), + ); + await app.register(fastifyCookie as never); + await app.register(fastifyMultipart as never); + await app.init(); + await app.getHttpAdapter().getInstance().ready(); + + await stopStagingWatcher(app); + + const db = app.get(DB); + const adminToken = await getAdminToken(app, db); + await setSetting(db, 'staging_auto_fetch_metadata', 'false'); + await setSetting(db, 'staging_auto_finalize_enabled', 'false'); + + return { + app, + db, + adminToken, + fixture, + envSnapshot, + }; +} + +export async function closeStagingE2EContext(ctx: StagingE2EContext): Promise { + await ctx.app.close(); + await ctx.fixture.cleanup(); + restoreEnv(ctx.envSnapshot); +} + +export async function resetStagingState(ctx: StagingE2EContext): Promise { + await ctx.db.delete(schema.stagingFiles); + await rm(ctx.fixture.stagingPath, { recursive: true, force: true }); + await mkdir(ctx.fixture.stagingPath, { recursive: true }); +} + +export async function createLibraryWithFolder( + ctx: StagingE2EContext, + options: { + mode?: 'book_per_file' | 'book_per_folder'; + allowedFormats?: string[]; + name?: string; + } = {}, +): Promise { + const folderPath = join(ctx.fixture.booksPath, `library-${randomUUID()}`); + await mkdir(folderPath, { recursive: true }); + + const { libraryId, libraryFolderId } = await seedLibrary(ctx.db, { + rootPath: folderPath, + mode: options.mode ?? 'book_per_folder', + allowedFormats: options.allowedFormats, + watch: false, + name: options.name, + }); + + return { + libraryId, + libraryFolderId, + folderPath, + }; +} + +export async function createUserAndLogin( + ctx: StagingE2EContext, + options: { + permissions?: Permission[]; + isSuperuser?: boolean; + username?: string; + password?: string; + email?: string; + } = {}, +): Promise { + const suffix = randomUUID().replaceAll('-', ''); + const username = options.username ?? `staging-user-${suffix}`; + const password = options.password ?? 'StagingUser123'; + const email = options.email ?? `${username}@example.com`; + const passwordHash = await hash(password, 12); + + const [created] = await ctx.db + .insert(schema.users) + .values({ + username, + name: `Staging User ${suffix}`, + email, + passwordHash, + isSuperuser: options.isSuperuser ?? false, + isDefaultPassword: false, + provisioningMethod: 'local', + }) + .returning({ id: schema.users.id }); + + const permissions = options.permissions ?? []; + if (permissions.length > 0) { + await ctx.db.insert(schema.userPermissions).values(permissions.map((permissionName) => ({ userId: created.id, permissionName }))); + } + + const loginResponse = await ctx.app.inject({ + method: 'POST', + url: '/api/v1/auth/login', + payload: { username, password }, + }); + + if (loginResponse.statusCode !== 200) { + throw new Error(`Login failed for ${username}: ${loginResponse.statusCode} ${loginResponse.body}`); + } + + const loginBody = loginResponse.json() as { accessToken?: string }; + if (!loginBody.accessToken) { + throw new Error(`Login for ${username} returned no accessToken`); + } + + return { + userId: created.id, + username, + password, + accessToken: loginBody.accessToken, + }; +} + +export async function grantLibraryAccess( + ctx: StagingE2EContext, + userId: number, + libraryId: number, + accessLevel: 'viewer' | 'editor' | 'owner' = 'viewer', +): Promise { + await ctx.db + .insert(schema.userLibraryAccess) + .values({ userId, libraryId, accessLevel }) + .onConflictDoUpdate({ + target: [schema.userLibraryAccess.userId, schema.userLibraryAccess.libraryId], + set: { accessLevel }, + }); +} + +export async function uploadStagingFile(ctx: StagingE2EContext, input: UploadStagingFileInput) { + const contentBuffer = typeof input.content === 'string' ? Buffer.from(input.content, 'utf8') : input.content; + const { body, boundary } = buildMultipartBody(input.fileName, contentBuffer, input.contentType ?? 'application/octet-stream'); + + return ctx.app.inject({ + method: 'POST', + url: '/api/v1/staging/upload', + headers: { + ...authHeader(input.token), + 'content-type': `multipart/form-data; boundary=${boundary}`, + 'content-length': String(body.length), + }, + payload: body, + }); +} + +export async function createStagingRow(ctx: StagingE2EContext, input: CreateStagingRowInput = {}): Promise { + const fileName = input.fileName ?? `staging-${randomUUID()}.fb2`; + const content = input.content ?? buildFb2Fixture({ title: `Title ${randomUUID()}`, authors: ['Fixture Author'] }); + const format = (input.format ?? extname(fileName).toLowerCase().slice(1)) || 'fb2'; + const relativePath = `staging/${fileName}`; + const absolutePath = await writeFixtureFile(ctx.fixture.booksPath, relativePath, content); + const fileStat = await stat(absolutePath); + + const [row] = await ctx.db + .insert(schema.stagingFiles) + .values({ + fileName, + absolutePath, + fileSize: fileStat.size, + format, + status: input.status ?? 'ready', + embeddedMetadata: input.embeddedMetadata ?? null, + selectedMetadata: input.selectedMetadata ?? null, + fetchedMetadata: input.fetchedMetadata ?? null, + targetLibraryId: input.targetLibraryId ?? null, + targetFolderId: input.targetFolderId ?? null, + confidence: input.confidence ?? null, + errorMessage: input.errorMessage ?? null, + metadataEditedAt: input.metadataEditedAt ?? null, + }) + .returning(); + + return row; +} + +export async function getStagingRow(ctx: StagingE2EContext, fileId: number) { + return ctx.db.query.stagingFiles.findFirst({ + where: eq(schema.stagingFiles.id, fileId), + }); +} + +export async function waitForStagingStatus( + ctx: StagingE2EContext, + fileId: number, + statuses: Array, + timeoutMs = 20_000, +): Promise { + let row: typeof schema.stagingFiles.$inferSelect | undefined; + + await waitForCondition(async () => { + row = await getStagingRow(ctx, fileId); + if (!row) { + throw new Error(`Staging file ${fileId} was not found`); + } + if (!statuses.includes(row.status)) { + throw new Error(`Staging file ${fileId} is ${row.status}, waiting for one of: ${statuses.join(', ')}`); + } + }, timeoutMs); + + return row!; +} + +export async function fileExists(path: string): Promise { + return stat(path) + .then(() => true) + .catch(() => false); +} + +function buildMultipartBody(fileName: string, content: Buffer, contentType: string): { body: Buffer; boundary: string } { + const boundary = `----projectx-staging-${randomUUID()}`; + const preamble = Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\nContent-Type: ${contentType}\r\n\r\n`, + 'utf8', + ); + const closing = Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8'); + return { body: Buffer.concat([preamble, content, closing]), boundary }; +} + +async function stopStagingWatcher(app: NestFastifyApplication): Promise { + const watcher = app.get(StagingWatcherService); + await watcher.onModuleDestroy(); +} + +async function getAdminToken(app: NestFastifyApplication, db: Db): Promise { + const response = await app.inject({ + method: 'POST', + url: '/api/v1/auth/setup', + payload: ADMIN_SETUP_DTO, + }); + + if (response.statusCode === 409) { + const existingToken = await loginForToken(app, ADMIN_SETUP_DTO.username, ADMIN_SETUP_DTO.password); + if (existingToken) return existingToken; + + const suffix = randomUUID().replaceAll('-', ''); + const fallbackUsername = `staging-e2e-admin-${suffix}`; + const fallbackPassword = ADMIN_SETUP_DTO.password; + const passwordHash = await hash(fallbackPassword, 12); + + await db.insert(schema.users).values({ + username: fallbackUsername, + name: 'Staging E2E Admin', + email: `staging-e2e-admin-${suffix}@example.com`, + passwordHash, + isSuperuser: true, + isDefaultPassword: false, + provisioningMethod: 'local', + }); + + const fallbackToken = await loginForToken(app, fallbackUsername, fallbackPassword); + if (fallbackToken) return fallbackToken; + throw new Error('Setup is already complete and fallback admin login failed'); + } + + if (response.statusCode !== 201) { + throw new Error(`Unable to complete setup: ${response.statusCode} ${response.body}`); + } + + const body = response.json() as { accessToken?: string }; + if (!body.accessToken) { + throw new Error('Setup succeeded but accessToken is missing'); + } + + return body.accessToken; +} + +async function loginForToken(app: NestFastifyApplication, username: string, password: string): Promise { + const response = await app.inject({ + method: 'POST', + url: '/api/v1/auth/login', + payload: { username, password }, + }); + if (response.statusCode !== 200) return null; + const body = response.json() as { accessToken?: string }; + return body.accessToken ?? null; +} + +async function setSetting(db: Db, key: string, value: string): Promise { + const updated = await db.update(schema.appSettings).set({ value }).where(eq(schema.appSettings.key, key)).returning({ id: schema.appSettings.id }); + if (updated.length === 0) { + throw new Error(`Missing app setting ${key}`); + } +} + +function restoreEnv(snapshot: EnvSnapshot): void { + if (snapshot.booksPath === undefined) delete process.env.BOOKS_PATH; + else process.env.BOOKS_PATH = snapshot.booksPath; + + if (snapshot.stagingPath === undefined) delete process.env.STAGING_PATH; + else process.env.STAGING_PATH = snapshot.stagingPath; +} diff --git a/server/test/staging-ingest-finalize.e2e-spec.ts b/server/test/staging-ingest-finalize.e2e-spec.ts new file mode 100644 index 000000000..937027792 --- /dev/null +++ b/server/test/staging-ingest-finalize.e2e-spec.ts @@ -0,0 +1,658 @@ +import { mkdir, writeFile } from 'fs/promises'; +import { dirname, join } from 'path'; +import { afterEach } from 'vitest'; + +import { eq, inArray } from 'drizzle-orm'; +import { Permission, type StagingFile, type StagingFinalizeResult } from '@projectx/types'; + +import * as schema from '../src/db/schema'; +import { buildFb2Fixture } from './e2e/staging/staging-fixture-builder'; +import { + authHeader, + closeStagingE2EContext, + createLibraryWithFolder, + createStagingE2EContext, + createStagingRow, + createUserAndLogin, + fileExists, + getStagingRow, + grantLibraryAccess, + resetStagingState, + uploadStagingFile, + waitForStagingStatus, + type StagingE2EContext, +} from './e2e/staging/staging-harness'; + +interface ScenarioRunResult { + id: string; + status: 'passed' | 'failed'; + durationMs: number; + error?: string; +} + +async function writeScenarioReport(results: ScenarioRunResult[]): Promise { + const reportDir = process.env.JUNIT_OUTPUT ? dirname(process.env.JUNIT_OUTPUT) : join(process.cwd(), '..', 'test-results', 'server'); + await mkdir(reportDir, { recursive: true }); + const reportPath = join(reportDir, 'staging-ingest-finalize-e2e-scenarios.json'); + await writeFile( + reportPath, + JSON.stringify( + { + generatedAt: new Date().toISOString(), + total: results.length, + passed: results.filter((result) => result.status === 'passed').length, + failed: results.filter((result) => result.status === 'failed').length, + results, + }, + null, + 2, + ), + ); +} + +describe('Staging ingest + finalize (e2e)', () => { + let context!: StagingE2EContext; + const scenarioResults: ScenarioRunResult[] = []; + let scenarioStartedAt = 0; + + beforeAll(async () => { + context = await createStagingE2EContext(); + }); + + afterEach((taskContext) => { + const result = taskContext.task.result; + if (!result) return; + + const state = result.state === 'pass' ? 'passed' : 'failed'; + const error = result.errors?.[0]?.message; + scenarioResults.push({ + id: taskContext.task.name, + status: state, + durationMs: Math.max(0, Date.now() - scenarioStartedAt), + ...(error ? { error } : {}), + }); + }); + + afterAll(async () => { + await writeScenarioReport(scenarioResults); + if (context) { + await closeStagingE2EContext(context); + } + }); + + beforeEach(async () => { + scenarioStartedAt = Date.now(); + await resetStagingState(context); + }); + + it('ingests upload, extracts metadata, and supports review editing', async () => { + const destination = await createLibraryWithFolder(context); + + const uploadResponse = await uploadStagingFile(context, { + token: context.adminToken, + fileName: 'ingest-review.fb2', + content: buildFb2Fixture({ + title: 'Ingest Review Title', + authors: ['Ingest Review Author'], + description: 'Review flow', + }), + contentType: 'application/xml', + }); + + expect(uploadResponse.statusCode).toBe(201); + const uploaded = uploadResponse.json() as StagingFile; + const ready = await waitForStagingStatus(context, uploaded.id, ['ready']); + + expect(ready.embeddedMetadata?.title).toBe('Ingest Review Title'); + expect(ready.embeddedMetadata?.authors).toEqual(['Ingest Review Author']); + expect(await fileExists(ready.absolutePath)).toBe(true); + + const patchResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/staging/files/${uploaded.id}`, + headers: authHeader(context.adminToken), + payload: { + selectedMetadata: { + title: 'Edited Review Title', + authors: ['Edited Review Author'], + genres: ['Mystery'], + }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }, + }); + + expect(patchResponse.statusCode).toBe(200); + const updated = patchResponse.json() as StagingFile; + expect(updated.selectedMetadata).toMatchObject({ + title: 'Edited Review Title', + authors: ['Edited Review Author'], + genres: ['Mystery'], + }); + expect(updated.targetLibraryId).toBe(destination.libraryId); + expect(updated.targetFolderId).toBe(destination.libraryFolderId); + + const summaryResponse = await context.app.inject({ + method: 'GET', + url: '/api/v1/staging/summary', + headers: authHeader(context.adminToken), + }); + expect(summaryResponse.statusCode).toBe(200); + expect(summaryResponse.json()).toMatchObject({ total: 1, ready: 1 }); + }); + + it('bulk set-target selectAll with status=pending includes pending, extracting, and fetching', async () => { + const destination = await createLibraryWithFolder(context); + const pendingRow = await createStagingRow(context, { fileName: 'pending-target.fb2', status: 'pending' }); + const extractingRow = await createStagingRow(context, { fileName: 'extracting-target.fb2', status: 'extracting' }); + const fetchingRow = await createStagingRow(context, { fileName: 'fetching-target.fb2', status: 'fetching' }); + const readyRow = await createStagingRow(context, { fileName: 'ready-target.fb2', status: 'ready' }); + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/set-target', + headers: authHeader(context.adminToken), + payload: { + selectAll: true, + excludedIds: [fetchingRow.id], + status: 'pending', + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }, + }); + + expect(response.statusCode).toBe(201); + expect(response.json()).toEqual({ total: 2, updated: 2, failed: 0 }); + + const rows = await context.db + .select({ + id: schema.stagingFiles.id, + targetLibraryId: schema.stagingFiles.targetLibraryId, + targetFolderId: schema.stagingFiles.targetFolderId, + }) + .from(schema.stagingFiles) + .where(inArray(schema.stagingFiles.id, [pendingRow.id, extractingRow.id, fetchingRow.id, readyRow.id])); + + const rowById = new Map(rows.map((row) => [row.id, row])); + expect(rowById.get(pendingRow.id)).toMatchObject({ + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + expect(rowById.get(extractingRow.id)).toMatchObject({ + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + expect(rowById.get(fetchingRow.id)).toMatchObject({ targetLibraryId: null, targetFolderId: null }); + expect(rowById.get(readyRow.id)).toMatchObject({ targetLibraryId: null, targetFolderId: null }); + }); + + it('bulk edit and apply fetched respect filters and metadataEditedAt', async () => { + const editableA = await createStagingRow(context, { + fileName: 'bulk-edit-target-a.fb2', + selectedMetadata: { authors: ['Existing Author'] }, + fetchedMetadata: { title: 'Fetched A', authors: ['Fetched Author A'] }, + }); + const editableB = await createStagingRow(context, { + fileName: 'bulk-apply-edited-b.fb2', + fetchedMetadata: { title: 'Fetched B' }, + metadataEditedAt: new Date(), + }); + const untouched = await createStagingRow(context, { + fileName: 'untouched-record.fb2', + selectedMetadata: { title: 'Untouched' }, + fetchedMetadata: { title: 'Fetched Untouched' }, + }); + + const bulkEditResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/bulk-edit', + headers: authHeader(context.adminToken), + payload: { + selectAll: true, + search: 'bulk-edit-target', + fields: { + title: 'Bulk Edited', + authors: ['Merged Author'], + }, + enabledFields: ['title', 'authors'], + mergeArrays: true, + }, + }); + + expect(bulkEditResponse.statusCode).toBe(201); + expect(bulkEditResponse.json()).toEqual({ total: 1, updated: 1, failed: 0 }); + + const applyFetchedResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/apply-fetched', + headers: authHeader(context.adminToken), + payload: { + fileIds: [editableA.id, editableB.id, untouched.id], + }, + }); + + expect(applyFetchedResponse.statusCode).toBe(201); + expect(applyFetchedResponse.json()).toEqual({ + total: 3, + applied: 2, + skipped: 0, + skippedEdited: 1, + }); + + const [rowA, rowB, rowUntouched] = await Promise.all([ + getStagingRow(context, editableA.id), + getStagingRow(context, editableB.id), + getStagingRow(context, untouched.id), + ]); + + expect(rowA?.selectedMetadata).toMatchObject({ title: 'Fetched A', authors: ['Fetched Author A'] }); + expect(rowB?.selectedMetadata ?? null).toBeNull(); + expect(rowUntouched?.selectedMetadata).toMatchObject({ title: 'Fetched Untouched' }); + }); + + it('bulk discard with selectAll + search honors excluded ids and removes files', async () => { + const removable = await createStagingRow(context, { fileName: 'discard-me-a.fb2' }); + const excluded = await createStagingRow(context, { fileName: 'discard-me-b.fb2' }); + const untouched = await createStagingRow(context, { fileName: 'keep-me.fb2' }); + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/discard', + headers: authHeader(context.adminToken), + payload: { + selectAll: true, + search: 'discard-me', + excludedIds: [excluded.id], + }, + }); + + expect(response.statusCode).toBe(204); + expect(await getStagingRow(context, removable.id)).toBeUndefined(); + expect(await getStagingRow(context, excluded.id)).toBeDefined(); + expect(await getStagingRow(context, untouched.id)).toBeDefined(); + + expect(await fileExists(removable.absolutePath)).toBe(false); + expect(await fileExists(excluded.absolutePath)).toBe(true); + expect(await fileExists(untouched.absolutePath)).toBe(true); + }); + + it('finalize moves files into destination and creates book records', async () => { + const destination = await createLibraryWithFolder(context); + const stagingRow = await createStagingRow(context, { + fileName: 'finalize-success.fb2', + selectedMetadata: { + title: 'Finalize Success Title', + authors: ['Finalize Author'], + }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(context.adminToken), + payload: { + fileIds: [stagingRow.id], + }, + }); + + expect(response.statusCode).toBe(201); + const body = response.json() as StagingFinalizeResult; + expect(body).toMatchObject({ total: 1, succeeded: 1, failed: 0 }); + expect(body.results[0]?.bookId).toEqual(expect.any(Number)); + + const finalizedBookId = body.results[0]!.bookId!; + + const [book] = await context.db + .select({ + id: schema.books.id, + libraryId: schema.books.libraryId, + }) + .from(schema.books) + .where(eq(schema.books.id, finalizedBookId)) + .limit(1); + expect(book).toMatchObject({ id: finalizedBookId, libraryId: destination.libraryId }); + + const [bookFile] = await context.db + .select({ + absolutePath: schema.bookFiles.absolutePath, + relPath: schema.bookFiles.relPath, + }) + .from(schema.bookFiles) + .where(eq(schema.bookFiles.bookId, finalizedBookId)) + .limit(1); + expect(bookFile).toBeDefined(); + expect(bookFile!.absolutePath.startsWith(destination.folderPath)).toBe(true); + expect(await fileExists(bookFile!.absolutePath)).toBe(true); + expect(await fileExists(stagingRow.absolutePath)).toBe(false); + + const [metadata] = await context.db + .select({ title: schema.bookMetadata.title }) + .from(schema.bookMetadata) + .where(eq(schema.bookMetadata.bookId, finalizedBookId)) + .limit(1); + expect(metadata?.title).toBe('Finalize Success Title'); + expect(await getStagingRow(context, stagingRow.id)).toBeUndefined(); + }); + + it('finalize returns partial success with duplicate and destination conflicts', async () => { + const destination = await createLibraryWithFolder(context); + + const seedRow = await createStagingRow(context, { + fileName: 'seed-duplicate.fb2', + selectedMetadata: { title: 'Duplicate Title', authors: ['Duplicate Author'] }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + const seedResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(context.adminToken), + payload: { fileIds: [seedRow.id] }, + }); + const seedBody = seedResponse.json() as StagingFinalizeResult; + const existingBookId = seedBody.results[0]!.bookId!; + + const duplicateRow = await createStagingRow(context, { + fileName: 'duplicate-input.fb2', + selectedMetadata: { title: 'Duplicate Title', authors: ['Another Author'] }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + const conflictRow = await createStagingRow(context, { + fileName: 'name-conflict.fb2', + selectedMetadata: { title: 'Conflict Title', authors: ['Conflict Author'] }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + const successRow = await createStagingRow(context, { + fileName: 'will-succeed.fb2', + selectedMetadata: { title: 'Success Title', authors: ['Success Author'] }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + + const previewNamesResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/preview-names', + headers: authHeader(context.adminToken), + payload: { + fileIds: [conflictRow.id], + defaultLibraryId: destination.libraryId, + }, + }); + expect(previewNamesResponse.statusCode).toBe(201); + const previewRows = previewNamesResponse.json() as Array<{ fileId: number; newName: string }>; + const conflictNewName = previewRows.find((row) => row.fileId === conflictRow.id)?.newName; + expect(conflictNewName).toEqual(expect.any(String)); + + const existingDestinationPath = join(destination.folderPath, conflictNewName!); + await mkdir(dirname(existingDestinationPath), { recursive: true }); + await writeFile(existingDestinationPath, 'already exists'); + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(context.adminToken), + payload: { + fileIds: [duplicateRow.id, conflictRow.id, successRow.id], + }, + }); + + expect(response.statusCode).toBe(201); + const body = response.json() as StagingFinalizeResult; + expect(body).toMatchObject({ total: 3, succeeded: 1, failed: 2 }); + + const duplicateResult = body.results.find((result) => result.fileId === duplicateRow.id); + const conflictResult = body.results.find((result) => result.fileId === conflictRow.id); + const successResult = body.results.find((result) => result.fileId === successRow.id); + + expect(duplicateResult).toMatchObject({ + success: false, + isDuplicate: true, + existingBookId, + }); + expect(conflictResult?.success).toBe(false); + expect(conflictResult?.message).toContain('already exists at the target location'); + expect(successResult?.success).toBe(true); + expect(successResult?.bookId).toEqual(expect.any(Number)); + + expect(await getStagingRow(context, duplicateRow.id)).toBeDefined(); + expect(await getStagingRow(context, conflictRow.id)).toBeDefined(); + expect(await getStagingRow(context, successRow.id)).toBeUndefined(); + }); + + it('finalize selectAll honors status/search filters and excluded ids', async () => { + const destination = await createLibraryWithFolder(context); + + const selected = await createStagingRow(context, { + fileName: 'batch-finalize-target-1.fb2', + selectedMetadata: { title: 'Batch Finalize 1' }, + status: 'ready', + }); + const excluded = await createStagingRow(context, { + fileName: 'batch-finalize-target-2.fb2', + selectedMetadata: { title: 'Batch Finalize 2' }, + status: 'ready', + }); + const unselected = await createStagingRow(context, { + fileName: 'other-finalize-target.fb2', + selectedMetadata: { title: 'Other Finalize Target' }, + status: 'ready', + }); + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(context.adminToken), + payload: { + selectAll: true, + status: 'ready', + search: 'batch-finalize-target', + excludedIds: [excluded.id], + defaultLibraryId: destination.libraryId, + defaultFolderId: destination.libraryFolderId, + }, + }); + + expect(response.statusCode).toBe(201); + expect(response.json()).toMatchObject({ total: 1, succeeded: 1, failed: 0 }); + expect(await getStagingRow(context, selected.id)).toBeUndefined(); + expect(await getStagingRow(context, excluded.id)).toBeDefined(); + expect(await getStagingRow(context, unselected.id)).toBeDefined(); + }); + + it('finalize marks missing file ids as failures instead of silently skipping them', async () => { + const destination = await createLibraryWithFolder(context); + const successfulRow = await createStagingRow(context, { + fileName: 'missing-id-success.fb2', + selectedMetadata: { title: 'Missing Id Success' }, + targetLibraryId: destination.libraryId, + targetFolderId: destination.libraryFolderId, + }); + const missingId = successfulRow.id + 50_000; + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(context.adminToken), + payload: { + fileIds: [successfulRow.id, missingId], + }, + }); + + expect(response.statusCode).toBe(201); + const body = response.json() as StagingFinalizeResult; + expect(body).toMatchObject({ total: 2, succeeded: 1, failed: 1 }); + expect(body.results.find((result) => result.fileId === missingId)).toMatchObject({ + success: false, + message: 'Staging file not found', + }); + expect(await getStagingRow(context, successfulRow.id)).toBeUndefined(); + }); + + it('enforces staging permission and library access for finalize', async () => { + const noPermissionUser = await createUserAndLogin(context); + const withPermissionUser = await createUserAndLogin(context, { + permissions: [Permission.StagingAccess], + }); + const destination = await createLibraryWithFolder(context); + const stagingRow = await createStagingRow(context, { + fileName: 'permission-check.fb2', + selectedMetadata: { title: 'Permission Check' }, + status: 'ready', + }); + + const forbiddenSummary = await context.app.inject({ + method: 'GET', + url: '/api/v1/staging/summary', + headers: authHeader(noPermissionUser.accessToken), + }); + expect(forbiddenSummary.statusCode).toBe(403); + + const allowedSummary = await context.app.inject({ + method: 'GET', + url: '/api/v1/staging/summary', + headers: authHeader(withPermissionUser.accessToken), + }); + expect(allowedSummary.statusCode).toBe(200); + + const noAccessFinalize = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(withPermissionUser.accessToken), + payload: { + fileIds: [stagingRow.id], + defaultLibraryId: destination.libraryId, + defaultFolderId: destination.libraryFolderId, + }, + }); + + expect(noAccessFinalize.statusCode).toBe(201); + const noAccessBody = noAccessFinalize.json() as StagingFinalizeResult; + expect(noAccessBody).toMatchObject({ total: 1, succeeded: 0, failed: 1 }); + expect(noAccessBody.results[0]?.message).toContain('No access to this library'); + expect(await getStagingRow(context, stagingRow.id)).toBeDefined(); + + await grantLibraryAccess(context, withPermissionUser.userId, destination.libraryId); + + const withAccessFinalize = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(withPermissionUser.accessToken), + payload: { + fileIds: [stagingRow.id], + defaultLibraryId: destination.libraryId, + defaultFolderId: destination.libraryFolderId, + }, + }); + + expect(withAccessFinalize.statusCode).toBe(201); + expect(withAccessFinalize.json()).toMatchObject({ total: 1, succeeded: 1, failed: 0 }); + expect(await getStagingRow(context, stagingRow.id)).toBeUndefined(); + }); + + it('rejects invalid request payloads and supports retry-fetch recovery', async () => { + const invalidPayloadResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/bulk-edit', + headers: authHeader(context.adminToken), + payload: { + fileIds: [123], + fields: { title: 'Invalid' }, + enabledFields: ['title'], + mergeArrays: false, + unexpected: true, + }, + }); + + expect(invalidPayloadResponse.statusCode).toBe(400); + const invalidBody = invalidPayloadResponse.json() as { message?: string[] }; + expect(invalidBody.message).toContain('property unexpected should not exist'); + + const targetValidationResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/set-target', + headers: authHeader(context.adminToken), + payload: { + fileIds: [], + targetLibraryId: 1, + }, + }); + + expect(targetValidationResponse.statusCode).toBe(400); + expect(targetValidationResponse.json()).toMatchObject({ + message: 'targetLibraryId and targetFolderId must both be set or both be null', + }); + + const retriableErrorRow = await createStagingRow(context, { + fileName: 'retry-fetch.fb2', + status: 'error', + errorMessage: 'Previous metadata failure', + content: buildFb2Fixture({ + title: 'Retry Fetch Title', + authors: ['Retry Author'], + }), + }); + const nonErrorRow = await createStagingRow(context, { + fileName: 'retry-ignore.fb2', + status: 'ready', + }); + + const retryResponse = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/files/retry-fetch', + headers: authHeader(context.adminToken), + payload: { + fileIds: [retriableErrorRow.id, nonErrorRow.id], + }, + }); + + expect(retryResponse.statusCode).toBe(201); + expect(retryResponse.json()).toEqual({ total: 2, queued: 1 }); + + const retried = await waitForStagingStatus(context, retriableErrorRow.id, ['ready']); + expect(retried.errorMessage).toBeNull(); + }); + + it('rejects unsupported upload formats', async () => { + const response = await uploadStagingFile(context, { + token: context.adminToken, + fileName: 'unsupported.txt', + content: 'not a supported book format', + contentType: 'text/plain', + }); + + expect(response.statusCode).toBe(400); + expect(response.json()).toMatchObject({ + message: expect.stringContaining('Unsupported file type .txt'), + }); + }); + + it('returns destination validation failure when folder does not belong to selected library', async () => { + const destinationA = await createLibraryWithFolder(context); + const destinationB = await createLibraryWithFolder(context); + const row = await createStagingRow(context, { + fileName: 'folder-mismatch.fb2', + selectedMetadata: { title: 'Folder Mismatch' }, + }); + + const response = await context.app.inject({ + method: 'POST', + url: '/api/v1/staging/finalize', + headers: authHeader(context.adminToken), + payload: { + fileIds: [row.id], + defaultLibraryId: destinationA.libraryId, + defaultFolderId: destinationB.libraryFolderId, + }, + }); + + expect(response.statusCode).toBe(201); + const body = response.json() as StagingFinalizeResult; + expect(body).toMatchObject({ total: 1, succeeded: 0, failed: 1 }); + expect(body.results[0]?.message).toContain('Folder does not belong to this library'); + expect(await getStagingRow(context, row.id)).toBeDefined(); + }); +});