fix(server): deduplicate migration user-state upsert batches (#81)

* fix(server): deduplicate migration user-state upsert batches

Prevent migration failures when source data contains duplicate user-state rows for the same target keys. Keep the latest row per upsert key and prefer richer reading progress signals on timestamp ties, then cover the behavior with focused unit and e2e tests.

Closes #71

* ci: raise commitlint body line limit to 500 chars

Allow longer commit body lines so convention checks do not fail on detailed but still concise commit explanations in this repository.

server/test/e2e/migration-booklore/migration-booklore-fixture-builder.ts

@@ -619,6 +619,34 @@ export async function seedCanonicalScenario(ctx: MigrationBookloreE2EContext): P
   };
 }
 
+export async function seedCanonicalScenarioWithDuplicateUserStateRows(ctx: MigrationBookloreE2EContext): Promise<CanonicalMigrationScenario> {
+  const scenario = await seedCanonicalScenario(ctx);
+
+  await withBookloreConnection(ctx, async (conn) => {
+    await insertRows(
+      conn,
+      'user_book_progress',
+      ['user_id', 'book_id', 'status', 'percentage', 'started_at', 'finished_at', 'updated_at'],
+      [
+        [1, 101, 'reading', 64, '2024-01-02 12:00:00', null, '2024-01-11 14:00:00'],
+        [2, 103, 'completed', 100, '2024-02-01 12:00:00', '2024-02-05 12:00:00', '2024-02-05 12:00:00'],
+      ],
+    );
+
+    await insertRows(
+      conn,
+      'user_book_file_progress',
+      ['user_id', 'book_id', 'book_file_id', 'percentage', 'cfi', 'position_href', 'page_number', 'position_seconds', 'updated_at'],
+      [
+        [1, 101, 1001, 44.4, 'epubcfi(/6/18!/4/2:10)', null, null, null, '2024-01-11 14:00:00'],
+        [2, 103, 1004, 77, null, null, null, 45, '2024-02-05 12:30:00'],
+      ],
+    );
+  });
+
+  return scenario;
+}
+
 export async function seedCompatibilityScenario(ctx: MigrationBookloreE2EContext): Promise<CompatibilityScenario> {
   const targetUser = await createUser(ctx, {
     username: 'migration-booklore-variant-source',

server/test/migration-booklore.e2e-spec.ts

@@ -18,6 +18,7 @@ import {
 } from './e2e/migration-booklore/migration-booklore-harness';
 import {
   seedCanonicalScenario,
+  seedCanonicalScenarioWithDuplicateUserStateRows,
   seedCompatibilityScenario,
   seedMissingRequiredTablesScenario,
   seedWarningsOnlyScenario,
@@ -60,7 +61,7 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
       token: ctx.adminToken,
     });
     expect(supportedTypes.statusCode).toBe(200);
-    expect(supportedTypes.body).toEqual(['booklore']);
+    expect(supportedTypes.body).toEqual(expect.arrayContaining(['booklore', 'grimmory']));
 
     const testedSource = await apiJson<{
       ok: boolean;
@@ -698,6 +699,126 @@ describe('Migration Booklore API to DB (e2e)', { timeout: 240_000 }, () => {
     await assertNoIntegrityViolations(ctx.db);
   });
 
+  it('handles duplicate source user-state rows without failing migration and keeps latest values', async () => {
+    const scenario = await seedCanonicalScenarioWithDuplicateUserStateRows(ctx);
+
+    const createdSource = await apiJson<{ id: number }>(ctx, {
+      method: 'POST',
+      url: '/api/v1/migration/sources',
+      token: ctx.adminToken,
+      payload: {
+        type: 'booklore',
+        name: 'Canonical With Duplicate User State',
+        connectionConfig: scenario.connectionConfig,
+      },
+    });
+    expect(createdSource.statusCode).toBe(201);
+
+    const validatedSource = await apiJson<{ ok: boolean }>(ctx, {
+      method: 'POST',
+      url: `/api/v1/migration/sources/${createdSource.body.id}/validate`,
+      token: ctx.adminToken,
+    });
+    expect(validatedSource.statusCode).toBe(200);
+    expect(validatedSource.body.ok).toBe(true);
+
+    const profile = await apiJson<{ id: number }>(ctx, {
+      method: 'POST',
+      url: '/api/v1/migration/profiles',
+      token: ctx.adminToken,
+      payload: {
+        sourceId: createdSource.body.id,
+        name: 'Duplicate User State Profile',
+        userMappings: [
+          { sourceUserId: '1', targetUserId: scenario.targetUsers.alice.id },
+          { sourceUserId: '2', targetUserId: scenario.targetUsers.bob.id },
+        ],
+        pathMappings: scenario.pathMappings,
+      },
+    });
+    expect(profile.statusCode).toBe(201);
+
+    const validatedMappings = await apiJson<{ persistedProfileId: number | null }>(ctx, {
+      method: 'POST',
+      url: `/api/v1/migration/sources/${createdSource.body.id}/path-mappings/validate`,
+      token: ctx.adminToken,
+      payload: {
+        pathMappings: scenario.pathMappings,
+      },
+    });
+    expect(validatedMappings.statusCode).toBe(201);
+    expect(validatedMappings.body.persistedProfileId).toBe(profile.body.id);
+
+    const dryRun = await apiJson<{
+      id: number;
+      plan: { duplicateBookMatches: Array<{ targetBookId: number; sourceBookIds: string[] }> };
+    }>(ctx, {
+      method: 'POST',
+      url: '/api/v1/migration/plans/dry-run',
+      token: ctx.adminToken,
+      payload: { profileId: profile.body.id },
+    });
+    expect(dryRun.statusCode).toBe(201);
+    expect(dryRun.body.plan.duplicateBookMatches).toHaveLength(1);
+
+    const resolvedPlan = await apiJson<{
+      id: number;
+      plan: { duplicateBookMatches: unknown[] };
+      summary: { status: string };
+    }>(ctx, {
+      method: 'POST',
+      url: `/api/v1/migration/plans/${dryRun.body.id}/resolve-duplicates`,
+      token: ctx.adminToken,
+      payload: {
+        resolutions: [{ targetBookId: scenario.books.duplicate.bookId, selectedSourceBookId: scenario.sourceBookIds.duplicatePreferred }],
+      },
+    });
+    expect(resolvedPlan.statusCode).toBe(200);
+    expect(resolvedPlan.body.plan.duplicateBookMatches).toEqual([]);
+    expect(resolvedPlan.body.summary.status).toBe('ready_for_live_run');
+
+    const startedRun = await apiJson<{ id: number; state: string }>(ctx, {
+      method: 'POST',
+      url: '/api/v1/migration/runs/live',
+      token: ctx.adminToken,
+      payload: { planArtifactId: resolvedPlan.body.id },
+    });
+    expect(startedRun.statusCode).toBe(201);
+    expect(startedRun.body.state).toBe('running');
+
+    const finished = await waitForMigrationToFinish(ctx, startedRun.body.id);
+    expect(finished.progress.run.state).toBe('completed');
+
+    const statusRows = await ctx.db.select().from(schema.userBookStatus);
+    expect(statusRows.find((row) => row.userId === scenario.targetUsers.alice.id && row.bookId === scenario.books.isbn.bookId)).toMatchObject({
+      status: 'reading',
+    });
+    expect(statusRows.find((row) => row.userId === scenario.targetUsers.bob.id && row.bookId === scenario.books.audio.bookId)).toMatchObject({
+      status: 'read',
+    });
+
+    const progressRows = await ctx.db.select().from(schema.readingProgress);
+    expect(
+      progressRows.find((row) => row.userId === scenario.targetUsers.alice.id && row.bookFileId === scenario.books.isbn.primaryFileId),
+    ).toMatchObject({
+      percentage: 44.4,
+      cfi: 'epubcfi(/6/18!/4/2:10)',
+    });
+    expect(
+      progressRows.find((row) => row.userId === scenario.targetUsers.bob.id && row.bookFileId === scenario.books.audio.fileIds[1]),
+    ).toMatchObject({
+      percentage: 77,
+      positionSeconds: 45,
+    });
+
+    const audioProgressRows = await ctx.db.select().from(schema.audiobookProgress);
+    expect(audioProgressRows.find((row) => row.userId === scenario.targetUsers.bob.id && row.bookId === scenario.books.audio.bookId)).toMatchObject({
+      percentage: 77,
+      currentFileId: scenario.books.audio.fileIds[1],
+      positionSeconds: 45,
+    });
+  });
+
   it('supports alternate Booklore schema variants and join-through file progress', async () => {
     const scenario = await seedCompatibilityScenario(ctx);