test(scanner): add staging ingest and finalize E2E test suite

This commit is contained in:
neon
2026-03-30 23:17:27 -06:00
parent 4d58418b93
commit 0c195165e6
8 changed files with 1231 additions and 8 deletions
@@ -0,0 +1,42 @@
name: Staging Ingest Finalize E2E
run-name: ${{ format('Staging Ingest Finalize E2E - {0}', github.ref_name) }}
on:
workflow_dispatch:
push:
paths:
- "server/src/modules/staging/**"
- "server/src/modules/upload/**"
- "server/src/modules/metadata/**"
- "server/test/staging-ingest-finalize.e2e-spec.ts"
- "server/test/e2e/staging/**"
- "server/src/db/schema/**"
- "scripts/test/**"
- "scripts/db/**"
pull_request:
paths:
- "server/src/modules/staging/**"
- "server/src/modules/upload/**"
- "server/src/modules/metadata/**"
- "server/test/staging-ingest-finalize.e2e-spec.ts"
- "server/test/e2e/staging/**"
- "server/src/db/schema/**"
- "scripts/test/**"
- "scripts/db/**"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
checks: write
jobs:
staging-ingest-finalize-e2e:
uses: ./.github/workflows/e2e-runner.yml
with:
suite-id: staging-ingest-finalize
check-name: Server Staging Ingest Finalize E2E
artifact-name: staging-ingest-finalize-e2e-report
job-timeout-minutes: 45
+15 -5
View File
@@ -118,6 +118,7 @@ pnpm run test:e2e -- all # run all e2e suites sequentially
pnpm run test:e2e -- smoke # e2e smoke suite (default DB)
pnpm run test:e2e -- scanner # scanner e2e suite (dedicated projectx_e2e DB)
pnpm run test:e2e -- scanner-file-ops # scanner file operation e2e suite (dedicated projectx_e2e DB)
pnpm run test:e2e -- staging-ingest-finalize # staging ingest/finalize e2e suite (dedicated projectx_e2e DB)
pnpm run test:e2e:all # alias for "pnpm run test:e2e -- all"
pnpm run test:e2e:list # list all supported e2e suite ids
```
@@ -158,6 +159,14 @@ pnpm run test:e2e -- scanner --testNamePattern=book-per-folder-disc-folder-flatt
- `scanner-file-ops-e2e-junit.xml`
- `scanner-file-ops-e2e-scenarios.json`
### Staging ingest/finalize e2e details
- `pnpm run test:e2e -- staging-ingest-finalize` prepares and migrates the dedicated e2e database (`projectx_e2e`) before running.
- Local runs auto-start PostgreSQL with `docker-compose.dev.yml` if needed.
- Results are written to `test-results/server/`:
- `staging-ingest-finalize-e2e-junit.xml`
- `staging-ingest-finalize-e2e-scenarios.json`
### E2E in CI (how to trigger)
Both workflows call the reusable `E2E Runner (reusable)` workflow, which runs:
@@ -166,15 +175,16 @@ Both workflows call the reusable `E2E Runner (reusable)` workflow, which runs:
pnpm run test:e2e -- <suite-id>
```
| Workflow | Suite id | Triggered by |
| -------------------- | ------------------ | -------------------------------------------------------------------------------- |
| Scanner E2E | `scanner` | `workflow_dispatch`, `push` (scanner-related paths), `pull_request` (same paths) |
| Scanner File Ops E2E | `scanner-file-ops` | `workflow_dispatch`, nightly schedule (`0 4 * * *`), `push`, `pull_request` |
| Workflow | Suite id | Triggered by |
| --------------------------- | ------------------------- | -------------------------------------------------------------------------------- |
| Scanner E2E | `scanner` | `workflow_dispatch`, `push` (scanner-related paths), `pull_request` (same paths) |
| Scanner File Ops E2E | `scanner-file-ops` | `workflow_dispatch`, nightly schedule (`0 4 * * *`), `push`, `pull_request` |
| Staging Ingest Finalize E2E | `staging-ingest-finalize` | `workflow_dispatch`, `push` (staging-related paths), `pull_request` (same paths) |
Manual trigger steps:
1. Open GitHub Actions.
2. Select **Scanner E2E** or **Scanner File Ops E2E**.
2. Select **Scanner E2E**, **Scanner File Ops E2E**, or **Staging Ingest Finalize E2E**.
3. Click **Run workflow**.
Each run uploads `test-results/server/` as an artifact and publishes JUnit annotations from `test-results/server/*-e2e-junit.xml`.
+1 -1
View File
@@ -171,7 +171,7 @@ onUnmounted(() => stopUploadCompletedListener())
<!-- Brand name -->
<div class="flex flex-col leading-none group-data-[collapsible=icon]:hidden">
<span class="text-lg font-serif font-semibold text-sidebar-foreground leading-tight tracking-tight">
Project<span class="text-primary"> X</span>
Project<span class="text-primary"> Q</span>
</span>
<span class="mt-0.5 text-[10px] uppercase tracking-[0.16em] text-sidebar-foreground/65">Library Control</span>
</div>
+8
View File
@@ -41,6 +41,14 @@ export const E2E_SUITES = Object.freeze({
prepareDedicatedDatabase: true,
useDedicatedDatabase: true,
},
"staging-ingest-finalize": {
id: "staging-ingest-finalize",
description: "Staging ingest and finalize suite",
vitestTarget: "test/staging-ingest-finalize.e2e-spec.ts",
junitOutput: `${TEST_RESULTS_DIR}/staging-ingest-finalize-e2e-junit.xml`,
prepareDedicatedDatabase: true,
useDedicatedDatabase: true,
},
});
export function listE2ESuites() {
@@ -76,7 +76,7 @@ export class StagingFinalizeService implements OnModuleInit {
status?: string,
search?: string,
): Promise<StagingFinalizeResult> {
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : (fileIds ?? []);
const ids = selectAll ? await this.repo.findAllIds(excludedIds, status, search) : dedupeIds(fileIds ?? []);
const overrideMap = new Map((overrides ?? []).map((o) => [o.fileId, o]));
const results: StagingFinalizeFileResult[] = [];
@@ -86,8 +86,21 @@ export class StagingFinalizeService implements OnModuleInit {
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
const batch = ids.slice(i, i + BATCH_SIZE);
const rows = await this.repo.findByIds(batch);
const rowById = new Map(rows.map((row) => [row.id, row]));
for (const fileId of batch) {
const row = rowById.get(fileId);
if (!row) {
failed++;
results.push({
fileId,
fileName: `staging-file-${fileId}`,
success: false,
message: 'Staging file not found',
});
continue;
}
for (const row of rows) {
const result = await this.finalizeFile(row, defaultLibraryId, defaultFolderId, overrideMap, userId, isSuperuser);
results.push(result);
if (result.success) succeeded++;
@@ -500,3 +513,7 @@ function shouldAutoFinalize(row: StagingFileRow, mode: StagingAutoFinalizeMetada
}
return row.confidence !== null && row.confidence >= threshold;
}
function dedupeIds(ids: number[]): number[] {
return [...new Set(ids)];
}
@@ -0,0 +1,93 @@
import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { dirname, join } from 'path';
export interface StagingFixtureRoot {
rootPath: string;
booksPath: string;
stagingPath: string;
cleanup: () => Promise<void>;
}
function assertRelativePath(path: string): void {
if (path.startsWith('/')) {
throw new Error(`Fixture paths must be relative. Received "${path}"`);
}
}
export async function createStagingFixtureRoot(prefix = 'staging-e2e-'): Promise<StagingFixtureRoot> {
const rootPath = await mkdtemp(join(tmpdir(), prefix));
const booksPath = join(rootPath, 'books');
const stagingPath = join(booksPath, 'staging');
await mkdir(stagingPath, { recursive: true });
return {
rootPath,
booksPath,
stagingPath,
cleanup: async () => {
await rm(rootPath, { recursive: true, force: true });
},
};
}
export async function writeFixtureFile(rootPath: string, relativePath: string, content: string | Buffer): Promise<string> {
assertRelativePath(relativePath);
const absolutePath = join(rootPath, relativePath);
await mkdir(dirname(absolutePath), { recursive: true });
await writeFile(absolutePath, content);
return absolutePath;
}
export interface Fb2FixtureInput {
title?: string;
authors?: string[];
language?: string;
genre?: string;
description?: string;
year?: number;
}
export function buildFb2Fixture(input: Fb2FixtureInput = {}): string {
const title = input.title ?? 'Staging Fixture Title';
const authors = input.authors ?? ['Staging Fixture Author'];
const language = input.language ?? 'en';
const genre = input.genre ?? 'fiction';
const description = input.description ?? 'Fixture description';
const year = input.year ?? 2024;
const authorXml = authors
.map((name) => {
const [firstName = 'Unknown', ...rest] = name.trim().split(/\s+/);
const lastName = rest.length > 0 ? rest.join(' ') : 'Author';
return `<author><first-name>${escapeXml(firstName)}</first-name><last-name>${escapeXml(lastName)}</last-name></author>`;
})
.join('');
return `<?xml version="1.0" encoding="utf-8"?>
<FictionBook xmlns:l="http://www.w3.org/1999/xlink">
<description>
<title-info>
<genre>${escapeXml(genre)}</genre>
${authorXml}
<book-title>${escapeXml(title)}</book-title>
<annotation><p>${escapeXml(description)}</p></annotation>
<date>${year}</date>
<lang>${escapeXml(language)}</lang>
</title-info>
<publish-info>
<year>${year}</year>
</publish-info>
</description>
<body>
<section><p>fixture content</p></section>
</body>
</FictionBook>
`;
}
function escapeXml(value: string): string {
return value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;').replaceAll("'", '&apos;');
}
+395
View File
@@ -0,0 +1,395 @@
import { randomUUID } from 'crypto';
import { hash } from 'bcryptjs';
import { extname, join } from 'path';
import { mkdir, rm, stat } from 'fs/promises';
import fastifyCookie from '@fastify/cookie';
import fastifyMultipart from '@fastify/multipart';
import { ValidationPipe } from '@nestjs/common';
import { Test } from '@nestjs/testing';
import { FastifyAdapter, type NestFastifyApplication } from '@nestjs/platform-fastify';
import { eq } from 'drizzle-orm';
import { Permission, type StagingMetadata } from '@projectx/types';
import { AppModule } from '../../../src/app.module';
import { DB } from '../../../src/db';
import * as schema from '../../../src/db/schema';
import { StagingWatcherService } from '../../../src/modules/staging/staging-watcher.service';
import { seedLibrary, waitForCondition, type Db } from '../app-harness';
import { buildFb2Fixture, createStagingFixtureRoot, writeFixtureFile, type StagingFixtureRoot } from './staging-fixture-builder';
const ADMIN_SETUP_DTO = {
username: 'staging-e2e-admin',
name: 'Staging E2E Admin',
email: 'staging-e2e-admin@example.com',
password: 'StagingAdmin123',
};
interface EnvSnapshot {
booksPath: string | undefined;
stagingPath: string | undefined;
}
export interface StagingE2EContext {
app: NestFastifyApplication;
db: Db;
adminToken: string;
fixture: StagingFixtureRoot;
envSnapshot: EnvSnapshot;
}
export interface TestUserSession {
userId: number;
username: string;
password: string;
accessToken: string;
}
export interface UploadStagingFileInput {
token: string;
fileName: string;
content: string | Buffer;
contentType?: string;
}
export interface CreateStagingRowInput {
fileName?: string;
content?: string | Buffer;
status?: typeof schema.stagingFiles.$inferInsert.status;
format?: string;
embeddedMetadata?: StagingMetadata | null;
selectedMetadata?: StagingMetadata | null;
fetchedMetadata?: StagingMetadata | null;
targetLibraryId?: number | null;
targetFolderId?: number | null;
confidence?: number | null;
errorMessage?: string | null;
metadataEditedAt?: Date | null;
}
export interface CreatedLibrary {
libraryId: number;
libraryFolderId: number;
folderPath: string;
}
export function authHeader(token: string): Record<string, string> {
return { authorization: `Bearer ${token}` };
}
export async function createStagingE2EContext(): Promise<StagingE2EContext> {
const fixture = await createStagingFixtureRoot();
const envSnapshot: EnvSnapshot = {
booksPath: process.env.BOOKS_PATH,
stagingPath: process.env.STAGING_PATH,
};
process.env.BOOKS_PATH = fixture.booksPath;
process.env.STAGING_PATH = fixture.stagingPath;
const moduleFixture = await Test.createTestingModule({
imports: [AppModule],
}).compile();
const app = moduleFixture.createNestApplication<NestFastifyApplication>(new FastifyAdapter());
app.setGlobalPrefix('api/v1');
app.useGlobalPipes(
new ValidationPipe({
whitelist: true,
forbidNonWhitelisted: true,
transform: true,
}),
);
await app.register(fastifyCookie as never);
await app.register(fastifyMultipart as never);
await app.init();
await app.getHttpAdapter().getInstance().ready();
await stopStagingWatcher(app);
const db = app.get<Db>(DB);
const adminToken = await getAdminToken(app, db);
await setSetting(db, 'staging_auto_fetch_metadata', 'false');
await setSetting(db, 'staging_auto_finalize_enabled', 'false');
return {
app,
db,
adminToken,
fixture,
envSnapshot,
};
}
export async function closeStagingE2EContext(ctx: StagingE2EContext): Promise<void> {
await ctx.app.close();
await ctx.fixture.cleanup();
restoreEnv(ctx.envSnapshot);
}
export async function resetStagingState(ctx: StagingE2EContext): Promise<void> {
await ctx.db.delete(schema.stagingFiles);
await rm(ctx.fixture.stagingPath, { recursive: true, force: true });
await mkdir(ctx.fixture.stagingPath, { recursive: true });
}
export async function createLibraryWithFolder(
ctx: StagingE2EContext,
options: {
mode?: 'book_per_file' | 'book_per_folder';
allowedFormats?: string[];
name?: string;
} = {},
): Promise<CreatedLibrary> {
const folderPath = join(ctx.fixture.booksPath, `library-${randomUUID()}`);
await mkdir(folderPath, { recursive: true });
const { libraryId, libraryFolderId } = await seedLibrary(ctx.db, {
rootPath: folderPath,
mode: options.mode ?? 'book_per_folder',
allowedFormats: options.allowedFormats,
watch: false,
name: options.name,
});
return {
libraryId,
libraryFolderId,
folderPath,
};
}
export async function createUserAndLogin(
ctx: StagingE2EContext,
options: {
permissions?: Permission[];
isSuperuser?: boolean;
username?: string;
password?: string;
email?: string;
} = {},
): Promise<TestUserSession> {
const suffix = randomUUID().replaceAll('-', '');
const username = options.username ?? `staging-user-${suffix}`;
const password = options.password ?? 'StagingUser123';
const email = options.email ?? `${username}@example.com`;
const passwordHash = await hash(password, 12);
const [created] = await ctx.db
.insert(schema.users)
.values({
username,
name: `Staging User ${suffix}`,
email,
passwordHash,
isSuperuser: options.isSuperuser ?? false,
isDefaultPassword: false,
provisioningMethod: 'local',
})
.returning({ id: schema.users.id });
const permissions = options.permissions ?? [];
if (permissions.length > 0) {
await ctx.db.insert(schema.userPermissions).values(permissions.map((permissionName) => ({ userId: created.id, permissionName })));
}
const loginResponse = await ctx.app.inject({
method: 'POST',
url: '/api/v1/auth/login',
payload: { username, password },
});
if (loginResponse.statusCode !== 200) {
throw new Error(`Login failed for ${username}: ${loginResponse.statusCode} ${loginResponse.body}`);
}
const loginBody = loginResponse.json() as { accessToken?: string };
if (!loginBody.accessToken) {
throw new Error(`Login for ${username} returned no accessToken`);
}
return {
userId: created.id,
username,
password,
accessToken: loginBody.accessToken,
};
}
export async function grantLibraryAccess(
ctx: StagingE2EContext,
userId: number,
libraryId: number,
accessLevel: 'viewer' | 'editor' | 'owner' = 'viewer',
): Promise<void> {
await ctx.db
.insert(schema.userLibraryAccess)
.values({ userId, libraryId, accessLevel })
.onConflictDoUpdate({
target: [schema.userLibraryAccess.userId, schema.userLibraryAccess.libraryId],
set: { accessLevel },
});
}
export async function uploadStagingFile(ctx: StagingE2EContext, input: UploadStagingFileInput) {
const contentBuffer = typeof input.content === 'string' ? Buffer.from(input.content, 'utf8') : input.content;
const { body, boundary } = buildMultipartBody(input.fileName, contentBuffer, input.contentType ?? 'application/octet-stream');
return ctx.app.inject({
method: 'POST',
url: '/api/v1/staging/upload',
headers: {
...authHeader(input.token),
'content-type': `multipart/form-data; boundary=${boundary}`,
'content-length': String(body.length),
},
payload: body,
});
}
export async function createStagingRow(ctx: StagingE2EContext, input: CreateStagingRowInput = {}): Promise<typeof schema.stagingFiles.$inferSelect> {
const fileName = input.fileName ?? `staging-${randomUUID()}.fb2`;
const content = input.content ?? buildFb2Fixture({ title: `Title ${randomUUID()}`, authors: ['Fixture Author'] });
const format = (input.format ?? extname(fileName).toLowerCase().slice(1)) || 'fb2';
const relativePath = `staging/${fileName}`;
const absolutePath = await writeFixtureFile(ctx.fixture.booksPath, relativePath, content);
const fileStat = await stat(absolutePath);
const [row] = await ctx.db
.insert(schema.stagingFiles)
.values({
fileName,
absolutePath,
fileSize: fileStat.size,
format,
status: input.status ?? 'ready',
embeddedMetadata: input.embeddedMetadata ?? null,
selectedMetadata: input.selectedMetadata ?? null,
fetchedMetadata: input.fetchedMetadata ?? null,
targetLibraryId: input.targetLibraryId ?? null,
targetFolderId: input.targetFolderId ?? null,
confidence: input.confidence ?? null,
errorMessage: input.errorMessage ?? null,
metadataEditedAt: input.metadataEditedAt ?? null,
})
.returning();
return row;
}
export async function getStagingRow(ctx: StagingE2EContext, fileId: number) {
return ctx.db.query.stagingFiles.findFirst({
where: eq(schema.stagingFiles.id, fileId),
});
}
export async function waitForStagingStatus(
ctx: StagingE2EContext,
fileId: number,
statuses: Array<typeof schema.stagingFiles.$inferSelect.status>,
timeoutMs = 20_000,
): Promise<typeof schema.stagingFiles.$inferSelect> {
let row: typeof schema.stagingFiles.$inferSelect | undefined;
await waitForCondition(async () => {
row = await getStagingRow(ctx, fileId);
if (!row) {
throw new Error(`Staging file ${fileId} was not found`);
}
if (!statuses.includes(row.status)) {
throw new Error(`Staging file ${fileId} is ${row.status}, waiting for one of: ${statuses.join(', ')}`);
}
}, timeoutMs);
return row!;
}
export async function fileExists(path: string): Promise<boolean> {
return stat(path)
.then(() => true)
.catch(() => false);
}
function buildMultipartBody(fileName: string, content: Buffer, contentType: string): { body: Buffer; boundary: string } {
const boundary = `----projectx-staging-${randomUUID()}`;
const preamble = Buffer.from(
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\nContent-Type: ${contentType}\r\n\r\n`,
'utf8',
);
const closing = Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8');
return { body: Buffer.concat([preamble, content, closing]), boundary };
}
async function stopStagingWatcher(app: NestFastifyApplication): Promise<void> {
const watcher = app.get(StagingWatcherService);
await watcher.onModuleDestroy();
}
async function getAdminToken(app: NestFastifyApplication, db: Db): Promise<string> {
const response = await app.inject({
method: 'POST',
url: '/api/v1/auth/setup',
payload: ADMIN_SETUP_DTO,
});
if (response.statusCode === 409) {
const existingToken = await loginForToken(app, ADMIN_SETUP_DTO.username, ADMIN_SETUP_DTO.password);
if (existingToken) return existingToken;
const suffix = randomUUID().replaceAll('-', '');
const fallbackUsername = `staging-e2e-admin-${suffix}`;
const fallbackPassword = ADMIN_SETUP_DTO.password;
const passwordHash = await hash(fallbackPassword, 12);
await db.insert(schema.users).values({
username: fallbackUsername,
name: 'Staging E2E Admin',
email: `staging-e2e-admin-${suffix}@example.com`,
passwordHash,
isSuperuser: true,
isDefaultPassword: false,
provisioningMethod: 'local',
});
const fallbackToken = await loginForToken(app, fallbackUsername, fallbackPassword);
if (fallbackToken) return fallbackToken;
throw new Error('Setup is already complete and fallback admin login failed');
}
if (response.statusCode !== 201) {
throw new Error(`Unable to complete setup: ${response.statusCode} ${response.body}`);
}
const body = response.json() as { accessToken?: string };
if (!body.accessToken) {
throw new Error('Setup succeeded but accessToken is missing');
}
return body.accessToken;
}
async function loginForToken(app: NestFastifyApplication, username: string, password: string): Promise<string | null> {
const response = await app.inject({
method: 'POST',
url: '/api/v1/auth/login',
payload: { username, password },
});
if (response.statusCode !== 200) return null;
const body = response.json() as { accessToken?: string };
return body.accessToken ?? null;
}
async function setSetting(db: Db, key: string, value: string): Promise<void> {
const updated = await db.update(schema.appSettings).set({ value }).where(eq(schema.appSettings.key, key)).returning({ id: schema.appSettings.id });
if (updated.length === 0) {
throw new Error(`Missing app setting ${key}`);
}
}
function restoreEnv(snapshot: EnvSnapshot): void {
if (snapshot.booksPath === undefined) delete process.env.BOOKS_PATH;
else process.env.BOOKS_PATH = snapshot.booksPath;
if (snapshot.stagingPath === undefined) delete process.env.STAGING_PATH;
else process.env.STAGING_PATH = snapshot.stagingPath;
}
@@ -0,0 +1,658 @@
import { mkdir, writeFile } from 'fs/promises';
import { dirname, join } from 'path';
import { afterEach } from 'vitest';
import { eq, inArray } from 'drizzle-orm';
import { Permission, type StagingFile, type StagingFinalizeResult } from '@projectx/types';
import * as schema from '../src/db/schema';
import { buildFb2Fixture } from './e2e/staging/staging-fixture-builder';
import {
authHeader,
closeStagingE2EContext,
createLibraryWithFolder,
createStagingE2EContext,
createStagingRow,
createUserAndLogin,
fileExists,
getStagingRow,
grantLibraryAccess,
resetStagingState,
uploadStagingFile,
waitForStagingStatus,
type StagingE2EContext,
} from './e2e/staging/staging-harness';
interface ScenarioRunResult {
id: string;
status: 'passed' | 'failed';
durationMs: number;
error?: string;
}
async function writeScenarioReport(results: ScenarioRunResult[]): Promise<void> {
const reportDir = process.env.JUNIT_OUTPUT ? dirname(process.env.JUNIT_OUTPUT) : join(process.cwd(), '..', 'test-results', 'server');
await mkdir(reportDir, { recursive: true });
const reportPath = join(reportDir, 'staging-ingest-finalize-e2e-scenarios.json');
await writeFile(
reportPath,
JSON.stringify(
{
generatedAt: new Date().toISOString(),
total: results.length,
passed: results.filter((result) => result.status === 'passed').length,
failed: results.filter((result) => result.status === 'failed').length,
results,
},
null,
2,
),
);
}
describe('Staging ingest + finalize (e2e)', () => {
let context!: StagingE2EContext;
const scenarioResults: ScenarioRunResult[] = [];
let scenarioStartedAt = 0;
beforeAll(async () => {
context = await createStagingE2EContext();
});
afterEach((taskContext) => {
const result = taskContext.task.result;
if (!result) return;
const state = result.state === 'pass' ? 'passed' : 'failed';
const error = result.errors?.[0]?.message;
scenarioResults.push({
id: taskContext.task.name,
status: state,
durationMs: Math.max(0, Date.now() - scenarioStartedAt),
...(error ? { error } : {}),
});
});
afterAll(async () => {
await writeScenarioReport(scenarioResults);
if (context) {
await closeStagingE2EContext(context);
}
});
beforeEach(async () => {
scenarioStartedAt = Date.now();
await resetStagingState(context);
});
it('ingests upload, extracts metadata, and supports review editing', async () => {
const destination = await createLibraryWithFolder(context);
const uploadResponse = await uploadStagingFile(context, {
token: context.adminToken,
fileName: 'ingest-review.fb2',
content: buildFb2Fixture({
title: 'Ingest Review Title',
authors: ['Ingest Review Author'],
description: 'Review flow',
}),
contentType: 'application/xml',
});
expect(uploadResponse.statusCode).toBe(201);
const uploaded = uploadResponse.json() as StagingFile;
const ready = await waitForStagingStatus(context, uploaded.id, ['ready']);
expect(ready.embeddedMetadata?.title).toBe('Ingest Review Title');
expect(ready.embeddedMetadata?.authors).toEqual(['Ingest Review Author']);
expect(await fileExists(ready.absolutePath)).toBe(true);
const patchResponse = await context.app.inject({
method: 'PATCH',
url: `/api/v1/staging/files/${uploaded.id}`,
headers: authHeader(context.adminToken),
payload: {
selectedMetadata: {
title: 'Edited Review Title',
authors: ['Edited Review Author'],
genres: ['Mystery'],
},
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
},
});
expect(patchResponse.statusCode).toBe(200);
const updated = patchResponse.json() as StagingFile;
expect(updated.selectedMetadata).toMatchObject({
title: 'Edited Review Title',
authors: ['Edited Review Author'],
genres: ['Mystery'],
});
expect(updated.targetLibraryId).toBe(destination.libraryId);
expect(updated.targetFolderId).toBe(destination.libraryFolderId);
const summaryResponse = await context.app.inject({
method: 'GET',
url: '/api/v1/staging/summary',
headers: authHeader(context.adminToken),
});
expect(summaryResponse.statusCode).toBe(200);
expect(summaryResponse.json()).toMatchObject({ total: 1, ready: 1 });
});
it('bulk set-target selectAll with status=pending includes pending, extracting, and fetching', async () => {
const destination = await createLibraryWithFolder(context);
const pendingRow = await createStagingRow(context, { fileName: 'pending-target.fb2', status: 'pending' });
const extractingRow = await createStagingRow(context, { fileName: 'extracting-target.fb2', status: 'extracting' });
const fetchingRow = await createStagingRow(context, { fileName: 'fetching-target.fb2', status: 'fetching' });
const readyRow = await createStagingRow(context, { fileName: 'ready-target.fb2', status: 'ready' });
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/set-target',
headers: authHeader(context.adminToken),
payload: {
selectAll: true,
excludedIds: [fetchingRow.id],
status: 'pending',
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
},
});
expect(response.statusCode).toBe(201);
expect(response.json()).toEqual({ total: 2, updated: 2, failed: 0 });
const rows = await context.db
.select({
id: schema.stagingFiles.id,
targetLibraryId: schema.stagingFiles.targetLibraryId,
targetFolderId: schema.stagingFiles.targetFolderId,
})
.from(schema.stagingFiles)
.where(inArray(schema.stagingFiles.id, [pendingRow.id, extractingRow.id, fetchingRow.id, readyRow.id]));
const rowById = new Map(rows.map((row) => [row.id, row]));
expect(rowById.get(pendingRow.id)).toMatchObject({
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
expect(rowById.get(extractingRow.id)).toMatchObject({
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
expect(rowById.get(fetchingRow.id)).toMatchObject({ targetLibraryId: null, targetFolderId: null });
expect(rowById.get(readyRow.id)).toMatchObject({ targetLibraryId: null, targetFolderId: null });
});
it('bulk edit and apply fetched respect filters and metadataEditedAt', async () => {
const editableA = await createStagingRow(context, {
fileName: 'bulk-edit-target-a.fb2',
selectedMetadata: { authors: ['Existing Author'] },
fetchedMetadata: { title: 'Fetched A', authors: ['Fetched Author A'] },
});
const editableB = await createStagingRow(context, {
fileName: 'bulk-apply-edited-b.fb2',
fetchedMetadata: { title: 'Fetched B' },
metadataEditedAt: new Date(),
});
const untouched = await createStagingRow(context, {
fileName: 'untouched-record.fb2',
selectedMetadata: { title: 'Untouched' },
fetchedMetadata: { title: 'Fetched Untouched' },
});
const bulkEditResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/bulk-edit',
headers: authHeader(context.adminToken),
payload: {
selectAll: true,
search: 'bulk-edit-target',
fields: {
title: 'Bulk Edited',
authors: ['Merged Author'],
},
enabledFields: ['title', 'authors'],
mergeArrays: true,
},
});
expect(bulkEditResponse.statusCode).toBe(201);
expect(bulkEditResponse.json()).toEqual({ total: 1, updated: 1, failed: 0 });
const applyFetchedResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/apply-fetched',
headers: authHeader(context.adminToken),
payload: {
fileIds: [editableA.id, editableB.id, untouched.id],
},
});
expect(applyFetchedResponse.statusCode).toBe(201);
expect(applyFetchedResponse.json()).toEqual({
total: 3,
applied: 2,
skipped: 0,
skippedEdited: 1,
});
const [rowA, rowB, rowUntouched] = await Promise.all([
getStagingRow(context, editableA.id),
getStagingRow(context, editableB.id),
getStagingRow(context, untouched.id),
]);
expect(rowA?.selectedMetadata).toMatchObject({ title: 'Fetched A', authors: ['Fetched Author A'] });
expect(rowB?.selectedMetadata ?? null).toBeNull();
expect(rowUntouched?.selectedMetadata).toMatchObject({ title: 'Fetched Untouched' });
});
it('bulk discard with selectAll + search honors excluded ids and removes files', async () => {
const removable = await createStagingRow(context, { fileName: 'discard-me-a.fb2' });
const excluded = await createStagingRow(context, { fileName: 'discard-me-b.fb2' });
const untouched = await createStagingRow(context, { fileName: 'keep-me.fb2' });
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/discard',
headers: authHeader(context.adminToken),
payload: {
selectAll: true,
search: 'discard-me',
excludedIds: [excluded.id],
},
});
expect(response.statusCode).toBe(204);
expect(await getStagingRow(context, removable.id)).toBeUndefined();
expect(await getStagingRow(context, excluded.id)).toBeDefined();
expect(await getStagingRow(context, untouched.id)).toBeDefined();
expect(await fileExists(removable.absolutePath)).toBe(false);
expect(await fileExists(excluded.absolutePath)).toBe(true);
expect(await fileExists(untouched.absolutePath)).toBe(true);
});
it('finalize moves files into destination and creates book records', async () => {
const destination = await createLibraryWithFolder(context);
const stagingRow = await createStagingRow(context, {
fileName: 'finalize-success.fb2',
selectedMetadata: {
title: 'Finalize Success Title',
authors: ['Finalize Author'],
},
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(context.adminToken),
payload: {
fileIds: [stagingRow.id],
},
});
expect(response.statusCode).toBe(201);
const body = response.json() as StagingFinalizeResult;
expect(body).toMatchObject({ total: 1, succeeded: 1, failed: 0 });
expect(body.results[0]?.bookId).toEqual(expect.any(Number));
const finalizedBookId = body.results[0]!.bookId!;
const [book] = await context.db
.select({
id: schema.books.id,
libraryId: schema.books.libraryId,
})
.from(schema.books)
.where(eq(schema.books.id, finalizedBookId))
.limit(1);
expect(book).toMatchObject({ id: finalizedBookId, libraryId: destination.libraryId });
const [bookFile] = await context.db
.select({
absolutePath: schema.bookFiles.absolutePath,
relPath: schema.bookFiles.relPath,
})
.from(schema.bookFiles)
.where(eq(schema.bookFiles.bookId, finalizedBookId))
.limit(1);
expect(bookFile).toBeDefined();
expect(bookFile!.absolutePath.startsWith(destination.folderPath)).toBe(true);
expect(await fileExists(bookFile!.absolutePath)).toBe(true);
expect(await fileExists(stagingRow.absolutePath)).toBe(false);
const [metadata] = await context.db
.select({ title: schema.bookMetadata.title })
.from(schema.bookMetadata)
.where(eq(schema.bookMetadata.bookId, finalizedBookId))
.limit(1);
expect(metadata?.title).toBe('Finalize Success Title');
expect(await getStagingRow(context, stagingRow.id)).toBeUndefined();
});
it('finalize returns partial success with duplicate and destination conflicts', async () => {
const destination = await createLibraryWithFolder(context);
const seedRow = await createStagingRow(context, {
fileName: 'seed-duplicate.fb2',
selectedMetadata: { title: 'Duplicate Title', authors: ['Duplicate Author'] },
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
const seedResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(context.adminToken),
payload: { fileIds: [seedRow.id] },
});
const seedBody = seedResponse.json() as StagingFinalizeResult;
const existingBookId = seedBody.results[0]!.bookId!;
const duplicateRow = await createStagingRow(context, {
fileName: 'duplicate-input.fb2',
selectedMetadata: { title: 'Duplicate Title', authors: ['Another Author'] },
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
const conflictRow = await createStagingRow(context, {
fileName: 'name-conflict.fb2',
selectedMetadata: { title: 'Conflict Title', authors: ['Conflict Author'] },
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
const successRow = await createStagingRow(context, {
fileName: 'will-succeed.fb2',
selectedMetadata: { title: 'Success Title', authors: ['Success Author'] },
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
const previewNamesResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/preview-names',
headers: authHeader(context.adminToken),
payload: {
fileIds: [conflictRow.id],
defaultLibraryId: destination.libraryId,
},
});
expect(previewNamesResponse.statusCode).toBe(201);
const previewRows = previewNamesResponse.json() as Array<{ fileId: number; newName: string }>;
const conflictNewName = previewRows.find((row) => row.fileId === conflictRow.id)?.newName;
expect(conflictNewName).toEqual(expect.any(String));
const existingDestinationPath = join(destination.folderPath, conflictNewName!);
await mkdir(dirname(existingDestinationPath), { recursive: true });
await writeFile(existingDestinationPath, 'already exists');
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(context.adminToken),
payload: {
fileIds: [duplicateRow.id, conflictRow.id, successRow.id],
},
});
expect(response.statusCode).toBe(201);
const body = response.json() as StagingFinalizeResult;
expect(body).toMatchObject({ total: 3, succeeded: 1, failed: 2 });
const duplicateResult = body.results.find((result) => result.fileId === duplicateRow.id);
const conflictResult = body.results.find((result) => result.fileId === conflictRow.id);
const successResult = body.results.find((result) => result.fileId === successRow.id);
expect(duplicateResult).toMatchObject({
success: false,
isDuplicate: true,
existingBookId,
});
expect(conflictResult?.success).toBe(false);
expect(conflictResult?.message).toContain('already exists at the target location');
expect(successResult?.success).toBe(true);
expect(successResult?.bookId).toEqual(expect.any(Number));
expect(await getStagingRow(context, duplicateRow.id)).toBeDefined();
expect(await getStagingRow(context, conflictRow.id)).toBeDefined();
expect(await getStagingRow(context, successRow.id)).toBeUndefined();
});
it('finalize selectAll honors status/search filters and excluded ids', async () => {
const destination = await createLibraryWithFolder(context);
const selected = await createStagingRow(context, {
fileName: 'batch-finalize-target-1.fb2',
selectedMetadata: { title: 'Batch Finalize 1' },
status: 'ready',
});
const excluded = await createStagingRow(context, {
fileName: 'batch-finalize-target-2.fb2',
selectedMetadata: { title: 'Batch Finalize 2' },
status: 'ready',
});
const unselected = await createStagingRow(context, {
fileName: 'other-finalize-target.fb2',
selectedMetadata: { title: 'Other Finalize Target' },
status: 'ready',
});
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(context.adminToken),
payload: {
selectAll: true,
status: 'ready',
search: 'batch-finalize-target',
excludedIds: [excluded.id],
defaultLibraryId: destination.libraryId,
defaultFolderId: destination.libraryFolderId,
},
});
expect(response.statusCode).toBe(201);
expect(response.json()).toMatchObject({ total: 1, succeeded: 1, failed: 0 });
expect(await getStagingRow(context, selected.id)).toBeUndefined();
expect(await getStagingRow(context, excluded.id)).toBeDefined();
expect(await getStagingRow(context, unselected.id)).toBeDefined();
});
it('finalize marks missing file ids as failures instead of silently skipping them', async () => {
const destination = await createLibraryWithFolder(context);
const successfulRow = await createStagingRow(context, {
fileName: 'missing-id-success.fb2',
selectedMetadata: { title: 'Missing Id Success' },
targetLibraryId: destination.libraryId,
targetFolderId: destination.libraryFolderId,
});
const missingId = successfulRow.id + 50_000;
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(context.adminToken),
payload: {
fileIds: [successfulRow.id, missingId],
},
});
expect(response.statusCode).toBe(201);
const body = response.json() as StagingFinalizeResult;
expect(body).toMatchObject({ total: 2, succeeded: 1, failed: 1 });
expect(body.results.find((result) => result.fileId === missingId)).toMatchObject({
success: false,
message: 'Staging file not found',
});
expect(await getStagingRow(context, successfulRow.id)).toBeUndefined();
});
it('enforces staging permission and library access for finalize', async () => {
const noPermissionUser = await createUserAndLogin(context);
const withPermissionUser = await createUserAndLogin(context, {
permissions: [Permission.StagingAccess],
});
const destination = await createLibraryWithFolder(context);
const stagingRow = await createStagingRow(context, {
fileName: 'permission-check.fb2',
selectedMetadata: { title: 'Permission Check' },
status: 'ready',
});
const forbiddenSummary = await context.app.inject({
method: 'GET',
url: '/api/v1/staging/summary',
headers: authHeader(noPermissionUser.accessToken),
});
expect(forbiddenSummary.statusCode).toBe(403);
const allowedSummary = await context.app.inject({
method: 'GET',
url: '/api/v1/staging/summary',
headers: authHeader(withPermissionUser.accessToken),
});
expect(allowedSummary.statusCode).toBe(200);
const noAccessFinalize = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(withPermissionUser.accessToken),
payload: {
fileIds: [stagingRow.id],
defaultLibraryId: destination.libraryId,
defaultFolderId: destination.libraryFolderId,
},
});
expect(noAccessFinalize.statusCode).toBe(201);
const noAccessBody = noAccessFinalize.json() as StagingFinalizeResult;
expect(noAccessBody).toMatchObject({ total: 1, succeeded: 0, failed: 1 });
expect(noAccessBody.results[0]?.message).toContain('No access to this library');
expect(await getStagingRow(context, stagingRow.id)).toBeDefined();
await grantLibraryAccess(context, withPermissionUser.userId, destination.libraryId);
const withAccessFinalize = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(withPermissionUser.accessToken),
payload: {
fileIds: [stagingRow.id],
defaultLibraryId: destination.libraryId,
defaultFolderId: destination.libraryFolderId,
},
});
expect(withAccessFinalize.statusCode).toBe(201);
expect(withAccessFinalize.json()).toMatchObject({ total: 1, succeeded: 1, failed: 0 });
expect(await getStagingRow(context, stagingRow.id)).toBeUndefined();
});
it('rejects invalid request payloads and supports retry-fetch recovery', async () => {
const invalidPayloadResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/bulk-edit',
headers: authHeader(context.adminToken),
payload: {
fileIds: [123],
fields: { title: 'Invalid' },
enabledFields: ['title'],
mergeArrays: false,
unexpected: true,
},
});
expect(invalidPayloadResponse.statusCode).toBe(400);
const invalidBody = invalidPayloadResponse.json() as { message?: string[] };
expect(invalidBody.message).toContain('property unexpected should not exist');
const targetValidationResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/set-target',
headers: authHeader(context.adminToken),
payload: {
fileIds: [],
targetLibraryId: 1,
},
});
expect(targetValidationResponse.statusCode).toBe(400);
expect(targetValidationResponse.json()).toMatchObject({
message: 'targetLibraryId and targetFolderId must both be set or both be null',
});
const retriableErrorRow = await createStagingRow(context, {
fileName: 'retry-fetch.fb2',
status: 'error',
errorMessage: 'Previous metadata failure',
content: buildFb2Fixture({
title: 'Retry Fetch Title',
authors: ['Retry Author'],
}),
});
const nonErrorRow = await createStagingRow(context, {
fileName: 'retry-ignore.fb2',
status: 'ready',
});
const retryResponse = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/files/retry-fetch',
headers: authHeader(context.adminToken),
payload: {
fileIds: [retriableErrorRow.id, nonErrorRow.id],
},
});
expect(retryResponse.statusCode).toBe(201);
expect(retryResponse.json()).toEqual({ total: 2, queued: 1 });
const retried = await waitForStagingStatus(context, retriableErrorRow.id, ['ready']);
expect(retried.errorMessage).toBeNull();
});
it('rejects unsupported upload formats', async () => {
const response = await uploadStagingFile(context, {
token: context.adminToken,
fileName: 'unsupported.txt',
content: 'not a supported book format',
contentType: 'text/plain',
});
expect(response.statusCode).toBe(400);
expect(response.json()).toMatchObject({
message: expect.stringContaining('Unsupported file type .txt'),
});
});
it('returns destination validation failure when folder does not belong to selected library', async () => {
const destinationA = await createLibraryWithFolder(context);
const destinationB = await createLibraryWithFolder(context);
const row = await createStagingRow(context, {
fileName: 'folder-mismatch.fb2',
selectedMetadata: { title: 'Folder Mismatch' },
});
const response = await context.app.inject({
method: 'POST',
url: '/api/v1/staging/finalize',
headers: authHeader(context.adminToken),
payload: {
fileIds: [row.id],
defaultLibraryId: destinationA.libraryId,
defaultFolderId: destinationB.libraryFolderId,
},
});
expect(response.statusCode).toBe(201);
const body = response.json() as StagingFinalizeResult;
expect(body).toMatchObject({ total: 1, succeeded: 0, failed: 1 });
expect(body.results[0]?.message).toContain('Folder does not belong to this library');
expect(await getStagingRow(context, row.id)).toBeDefined();
});
});